From 7177207b00d5e68d7c0e8e251140e32b3901dfff Mon Sep 17 00:00:00 2001 From: Alisha Date: Tue, 15 Oct 2024 10:46:55 -0700 Subject: [PATCH 01/32] update generators to have optional params --- .../transformers/generate_card_number.go | 2 +- .../transformers/generate_card_number_test.go | 11 ++++++++++ .../pkg/benthos/transformers/generate_city.go | 2 +- .../transformers/generate_city_test.go | 13 +++++++++++ .../transformers/generate_country_test.go | 22 +++++++++++++++++++ .../transformers/generate_email_test.go | 18 +++++++++++++++ .../transformers/generate_first_name.go | 2 +- .../transformers/generate_first_name_test.go | 15 +++++++++++++ .../benthos/transformers/generate_float.go | 4 ++-- .../transformers/generate_float_test.go | 10 +++++++++ .../transformers/generate_full_address.go | 2 +- .../generate_full_address_test.go | 10 +++++++++ .../transformers/generate_full_name.go | 2 +- .../transformers/generate_full_name_test.go | 11 ++++++++++ 14 files changed, 117 insertions(+), 7 deletions(-) diff --git a/worker/pkg/benthos/transformers/generate_card_number.go b/worker/pkg/benthos/transformers/generate_card_number.go index de77c2c7d3..d851c7a2c1 100644 --- a/worker/pkg/benthos/transformers/generate_card_number.go +++ b/worker/pkg/benthos/transformers/generate_card_number.go @@ -17,7 +17,7 @@ const defaultIIN = 400000 func init() { spec := bloblang.NewPluginSpec(). Description("Generates a 16 digit card number that is valid by Luhn valid by default."). - Param(bloblang.NewBoolParam("valid_luhn").Description("A boolean indicating whether the generated value should pass the Luhn algorithm check.")). + Param(bloblang.NewBoolParam("valid_luhn").Default(false).Description("A boolean indicating whether the generated value should pass the Luhn algorithm check.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_card_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_card_number_test.go b/worker/pkg/benthos/transformers/generate_card_number_test.go index 5ae5b28a77..3172616c96 100644 --- a/worker/pkg/benthos/transformers/generate_card_number_test.go +++ b/worker/pkg/benthos/transformers/generate_card_number_test.go @@ -37,6 +37,17 @@ func Test_GenerateCardNumberTransformer(t *testing.T) { assert.Equal(t, true, isValidLuhn(res.(int64)), "The output card number should pass luhn validation") } +func Test_GenerateCardNumberTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_card_number()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the random int transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + assert.Len(t, strconv.FormatInt(res.(int64), 10), 16, "The output card should be 16 characters long") +} + func isValidLuhn(cc int64) bool { return (cc%10+checksum(cc/10))%10 == 0 } diff --git a/worker/pkg/benthos/transformers/generate_city.go b/worker/pkg/benthos/transformers/generate_city.go index 355a42cd0a..799cfe27a8 100644 --- a/worker/pkg/benthos/transformers/generate_city.go +++ b/worker/pkg/benthos/transformers/generate_city.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Randomly selects a city from a list of predefined US cities."). - Param(bloblang.NewInt64Param("max_length").Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_city", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_city_test.go b/worker/pkg/benthos/transformers/generate_city_test.go index 5042bb25c6..7a0aae01a8 100644 --- a/worker/pkg/benthos/transformers/generate_city_test.go +++ b/worker/pkg/benthos/transformers/generate_city_test.go @@ -40,3 +40,16 @@ func Test_CityTransformer(t *testing.T) { assert.LessOrEqual(t, int64(len(res.(string))), maxLength, fmt.Sprintf("The city should be less than or equal to the max length. This is the error city:%s", res)) } + +func Test_CityTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_city()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the city transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + assert.IsType(t, "", res, "The returned city should be a string") + + assert.NotEmptyf(t, res, fmt.Sprintf("The city should be not be empty. This is the error city:%s", res)) +} diff --git a/worker/pkg/benthos/transformers/generate_country_test.go b/worker/pkg/benthos/transformers/generate_country_test.go index 6e6f39fdfd..9cc536d5b4 100644 --- a/worker/pkg/benthos/transformers/generate_country_test.go +++ b/worker/pkg/benthos/transformers/generate_country_test.go @@ -81,3 +81,25 @@ func Test_CountryTransformer(t *testing.T) { assert.Len(t, res, 2) assert.True(t, countryExists, "The generated country should exist in the countrys.go file") } + +func Test_CountryTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_country()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the country transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + assert.IsType(t, Address{}.City, res, "The returned country should be a string") + + countryExists := false + for _, country := range transformers_dataset.Countries { + if country.Code == res { + countryExists = true + break + } + } + + assert.Len(t, res, 2) + assert.True(t, countryExists, "The generated country should exist in the countrys.go file") +} diff --git a/worker/pkg/benthos/transformers/generate_email_test.go b/worker/pkg/benthos/transformers/generate_email_test.go index 22910d29c5..d1d55ec61f 100644 --- a/worker/pkg/benthos/transformers/generate_email_test.go +++ b/worker/pkg/benthos/transformers/generate_email_test.go @@ -73,3 +73,21 @@ func Test_RandomEmailTransformer(t *testing.T) { require.Equal(t, true, transformer_utils.IsValidEmail(res.(string)), "The expected email should have a valid email format") } + +func Test_RandomEmailTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_email()` + ex, err := bloblang.Parse(mapping) + require.NoError(t, err) + require.NotEmpty(t, ex) + + res, err := ex.Query(nil) + require.NoError(t, err) + require.NotEmpty(t, res) + + resStr, ok := res.(string) + require.True(t, ok) + require.NotEmpty(t, resStr) + + require.NotEmptyf(t, resStr, fmt.Sprintf("The email should be less than or equal to the max length. This is the error email:%s", res)) + +} diff --git a/worker/pkg/benthos/transformers/generate_first_name.go b/worker/pkg/benthos/transformers/generate_first_name.go index 3bf4c7aac4..64cd647027 100644 --- a/worker/pkg/benthos/transformers/generate_first_name.go +++ b/worker/pkg/benthos/transformers/generate_first_name.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random first name between 2 and 12 characters long."). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_first_name", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_first_name_test.go b/worker/pkg/benthos/transformers/generate_first_name_test.go index d5d771eb21..115b914fd7 100644 --- a/worker/pkg/benthos/transformers/generate_first_name_test.go +++ b/worker/pkg/benthos/transformers/generate_first_name_test.go @@ -56,3 +56,18 @@ func Test_GenerateRandomFirstNameTransformer(t *testing.T) { assert.NotEmpty(t, resStr) assert.LessOrEqual(t, int64(len(resStr)), maxCharacterLimit, "output should be less than or equal to max char limit") } + +func Test_GenerateRandomFirstNameTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_first_name()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the first name transformer") + assert.NotNil(t, ex) + + res, err := ex.Query(nil) + assert.NoError(t, err) + + resStr, ok := res.(string) + require.True(t, ok) + + assert.NotEmpty(t, resStr) +} diff --git a/worker/pkg/benthos/transformers/generate_float.go b/worker/pkg/benthos/transformers/generate_float.go index c4e5df7466..90f2249e01 100644 --- a/worker/pkg/benthos/transformers/generate_float.go +++ b/worker/pkg/benthos/transformers/generate_float.go @@ -17,8 +17,8 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random floating point number with a max precision of 17. Go float64 adheres to the IEEE 754 standard for double-precision floating-point numbers."). Param(bloblang.NewBoolParam("randomize_sign").Default(false).Description("A boolean indicating whether the sign of the float should be randomized.")). - Param(bloblang.NewFloat64Param("min").Description("Specifies the minimum value for the generated float.")). - Param(bloblang.NewFloat64Param("max").Description("Specifies the maximum value for the generated float")). + Param(bloblang.NewFloat64Param("min").Default(1).Description("Specifies the minimum value for the generated float.")). + Param(bloblang.NewFloat64Param("max").Default(10000).Description("Specifies the maximum value for the generated float")). Param(bloblang.NewInt64Param("precision").Optional().Description("An optional parameter that defines the number of significant digits for the generated float.")). Param(bloblang.NewInt64Param("scale").Optional().Description("An optional parameter that defines the number of decimal places for the generated float.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) diff --git a/worker/pkg/benthos/transformers/generate_float_test.go b/worker/pkg/benthos/transformers/generate_float_test.go index 69bdec4f94..6a2fb78110 100644 --- a/worker/pkg/benthos/transformers/generate_float_test.go +++ b/worker/pkg/benthos/transformers/generate_float_test.go @@ -91,3 +91,13 @@ func Test_GenerateRandomFloat_Benthos(t *testing.T) { assert.GreaterOrEqual(t, res.(float64), minValue, "The result should be greater or equal to the minimum") assert.LessOrEqual(t, res.(float64), maxValue, "The result should be less or equal to the maximum") } + +func Test_GenerateRandomFloat_Benthos_NoOptions(t *testing.T) { + mapping := `root = generate_float64()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the generate float transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_full_address.go b/worker/pkg/benthos/transformers/generate_full_address.go index a5805109fa..5374f8fe68 100644 --- a/worker/pkg/benthos/transformers/generate_full_address.go +++ b/worker/pkg/benthos/transformers/generate_full_address.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a randomly selected real full address that exists in the United States."). - Param(bloblang.NewInt64Param("max_length").Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_full_address", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_full_address_test.go b/worker/pkg/benthos/transformers/generate_full_address_test.go index 46baad5e76..88b98ca3d2 100644 --- a/worker/pkg/benthos/transformers/generate_full_address_test.go +++ b/worker/pkg/benthos/transformers/generate_full_address_test.go @@ -51,3 +51,13 @@ func Test_FullAddressTransformer(t *testing.T) { assert.IsType(t, "", res, "The returned full address should be a string") assert.LessOrEqual(t, int64(len(res.(string))), faMaxLength, fmt.Sprintf("The city should be less than or equal to the max length. This is the error address:%s", res)) } + +func Test_FullAddressTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_full_address()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the full address transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_full_name.go b/worker/pkg/benthos/transformers/generate_full_name.go index f1adcac246..f7216329a4 100644 --- a/worker/pkg/benthos/transformers/generate_full_name.go +++ b/worker/pkg/benthos/transformers/generate_full_name.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a new full name consisting of a first and last name."). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_full_name", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_full_name_test.go b/worker/pkg/benthos/transformers/generate_full_name_test.go index 5c63a3b860..33db84648f 100644 --- a/worker/pkg/benthos/transformers/generate_full_name_test.go +++ b/worker/pkg/benthos/transformers/generate_full_name_test.go @@ -55,3 +55,14 @@ func Test_GenerateRandomFullNameTransformer(t *testing.T) { assert.NotEmpty(t, resStr) assert.LessOrEqual(t, int64(len(resStr)), maxCharacterLimit, "output should be less than or equal to max char limit") } + +func Test_GenerateRandomFullNameTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_full_name()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the first name transformer") + assert.NotNil(t, ex) + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} From 0d4a46c495107ae834a8b138c627bb172678b457 Mon Sep 17 00:00:00 2001 From: Alisha Date: Tue, 15 Oct 2024 15:48:17 -0700 Subject: [PATCH 02/32] make transformer arguments optional --- .../gen-javascript-transformer.md | 128 +++++++-------- .../apps/web/@types/neosync-transformers.d.ts | 48 +++--- .../benthos/transformers/gen_generate_bool.go | 2 +- .../transformers/gen_generate_card_number.go | 15 +- .../transformers/gen_generate_categorical.go | 15 +- .../benthos/transformers/gen_generate_city.go | 15 +- .../transformers/gen_generate_country.go | 4 +- .../transformers/gen_generate_email.go | 6 +- .../transformers/gen_generate_first_name.go | 8 +- .../transformers/gen_generate_float.go | 30 ++-- .../transformers/gen_generate_full_address.go | 15 +- .../transformers/gen_generate_full_name.go | 8 +- .../transformers/gen_generate_gender.go | 6 +- .../transformers/gen_generate_int64.go | 30 ++-- .../gen_generate_int64_phone_number.go | 2 +- ...gen_generate_international_phone_number.go | 28 ++-- .../transformers/gen_generate_last_name.go | 8 +- .../gen_generate_random_string.go | 28 ++-- .../benthos/transformers/gen_generate_ssn.go | 2 +- .../transformers/gen_generate_state.go | 4 +- .../gen_generate_street_address.go | 15 +- .../gen_generate_string_phone_number.go | 28 ++-- .../gen_generate_unix_timestamp.go | 2 +- .../transformers/gen_generate_username.go | 8 +- .../gen_generate_utc_timestamp.go | 2 +- .../benthos/transformers/gen_generate_uuid.go | 3 +- .../transformers/gen_generate_zipcode.go | 2 +- .../gen_transform_character_scramble.go | 2 +- .../gen_transform_e164_phone_number.go | 15 +- .../transformers/gen_transform_email.go | 18 +-- .../transformers/gen_transform_first_name.go | 10 +- .../transformers/gen_transform_float.go | 28 ++-- .../transformers/gen_transform_full_name.go | 10 +- .../transformers/gen_transform_int64.go | 28 ++-- .../gen_transform_int64_phone_number.go | 15 +- .../transformers/gen_transform_lastname.go | 10 +- .../transformers/gen_transform_string.go | 12 +- .../gen_transform_string_phone_number.go | 28 ++-- .../transformers/generate_categorical.go | 2 +- .../transformers/generate_gender_test.go | 10 ++ .../benthos/transformers/generate_int64.go | 4 +- .../transformers/generate_int64_test.go | 10 ++ .../generate_international_phone_number.go | 4 +- ...enerate_international_phone_number_test.go | 10 ++ .../transformers/generate_last_name.go | 2 +- .../transformers/generate_last_name_test.go | 11 ++ .../transformers/generate_random_string.go | 4 +- .../generate_random_string_test.go | 10 ++ .../transformers/generate_state_test.go | 11 ++ .../transformers/generate_street_address.go | 2 +- .../generate_street_address_test.go | 10 ++ .../generate_string_phone_number.go | 4 +- .../generate_string_phone_number_test.go | 10 ++ .../benthos/transformers/generate_username.go | 2 +- .../transformers/generate_username_test.go | 11 ++ .../transformers/generate_uuid_test.go | 10 ++ .../neosync_transformer_generator.go | 4 +- .../transform_e164_phone_number.go | 2 +- .../transform_e164_phone_number_test.go | 11 ++ .../benthos/transformers/transform_email.go | 2 +- .../transformers/transform_email_test.go | 10 ++ .../transformers/transform_first_name.go | 2 +- .../transformers/transform_first_name_test.go | 10 ++ .../benthos/transformers/transform_float.go | 4 +- .../transformers/transform_float_test.go | 11 ++ .../transformers/transform_full_name.go | 2 +- .../transformers/transform_full_name_test.go | 10 ++ .../benthos/transformers/transform_int64.go | 4 +- .../transform_int64_phone_number.go | 2 +- .../transform_int64_phone_number_test.go | 10 ++ .../transformers/transform_int64_test.go | 11 ++ .../transformers/transform_lastname.go | 2 +- .../transformers/transform_lastname_test.go | 10 ++ .../benthos/transformers/transform_string.go | 2 +- .../transform_string_phone_number.go | 4 +- .../transform_string_phone_number_test.go | 10 ++ .../transformers/transform_string_test.go | 10 ++ .../transformers/transformer_initializer.go | 55 +++---- .../transformer_initializer_test.go | 152 ++++++++++++++++++ .../benthos/transformers/utils/type_utils.go | 19 +++ .../transformers/utils/type_utils_test.go | 77 +++++++++ 81 files changed, 884 insertions(+), 317 deletions(-) create mode 100644 worker/pkg/benthos/transformers/transformer_initializer_test.go create mode 100644 worker/pkg/benthos/transformers/utils/type_utils.go create mode 100644 worker/pkg/benthos/transformers/utils/type_utils_test.go diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index 2b949b73d6..00b87ecb9a 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -77,7 +77,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| preserveLength | bool | | true | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. +| preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | maxLength | int64 | | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -86,7 +86,7 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformE164PhoneNumber(value, { - preserveLength: false, + preserveLength: false, maxLength: 1, seed: 1, }); @@ -116,7 +116,7 @@ Description: Value that will be transformed | preserveLength | bool | false | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | preserveDomain | bool | false | false | A boolean indicating whether the domain part of the email should be preserved. | excludedDomains | any | [] | false | A list of domains that should be excluded from the transformation -| maxLength | int64 | 10000 | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. +| maxLength | int64 | 100 | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | seed | int64 | | false | An optional seed value used for generating deterministic transformations. | emailType | string | 'uuidv4' | false | Specifies the type of email to transform, with options including `uuidv4`, `fullname`, or `any`. | invalidEmailAction | string | 'reject' | false | Specifies the action to take when an invalid email is encountered, with options including `reject`, `passthrough`, `null`, or `generate`.
@@ -129,7 +129,7 @@ const newValue = neosync.transformEmail(value, { preserveLength: false, preserveDomain: false, excludedDomains: [], - maxLength: 10000, + maxLength: 100, seed: 1, emailType: 'uuidv4', invalidEmailAction: 'reject', @@ -157,7 +157,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | seed | int64 | | false | An optional seed value used for generating deterministic transformations.
@@ -166,7 +166,7 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformFirstName(value, { - maxLength: 10000, + maxLength: 100, preserveLength: false, seed: 1, }); @@ -193,8 +193,8 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| randomizationRangeMin | float64 | | true | Specifies the minimum value for the range of the float. -| randomizationRangeMax | float64 | | true | Specifies the maximum value for the randomization range of the float. +| randomizationRangeMin | float64 | 1 | false | Specifies the minimum value for the range of the float. +| randomizationRangeMax | float64 | 10000 | false | Specifies the maximum value for the randomization range of the float. | precision | int64 | | false | An optional parameter that defines the number of significant digits for the float. | scale | int64 | | false | An optional parameter that defines the number of decimal places for the float. | seed | int64 | | false | An optional seed value used for generating deterministic transformations.
@@ -204,8 +204,8 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformFloat64(value, { - randomizationRangeMin: 1.12, - randomizationRangeMax: 1.12, + randomizationRangeMin: 1, + randomizationRangeMax: 10000, precision: 1, scale: 1, seed: 1, @@ -233,7 +233,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | seed | int64 | | false | An optional seed value used for generating deterministic transformations.
@@ -242,7 +242,7 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformFullName(value, { - maxLength: 10000, + maxLength: 100, preserveLength: false, seed: 1, }); @@ -269,8 +269,8 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| randomizationRangeMin | int64 | | true | Specifies the minimum value for the range of the int. -| randomizationRangeMax | int64 | | true | Specifies the maximum value for the range of the int. +| randomizationRangeMin | int64 | 1 | false | Specifies the minimum value for the range of the int. +| randomizationRangeMax | int64 | 10000 | false | Specifies the maximum value for the range of the int. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** @@ -278,8 +278,8 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformInt64(value, { - randomizationRangeMin: 1, - randomizationRangeMax: 1, + randomizationRangeMin: 1, + randomizationRangeMax: 10000, seed: 1, }); @@ -305,7 +305,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| preserveLength | bool | | true | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. +| preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** @@ -313,7 +313,7 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformInt64PhoneNumber(value, { - preserveLength: false, + preserveLength: false, seed: 1, }); @@ -339,7 +339,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | seed | int64 | | false | An optional seed value used for generating deterministic transformations.
@@ -348,7 +348,7 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformLastName(value, { - maxLength: 10000, + maxLength: 100, preserveLength: false, seed: 1, }); @@ -377,7 +377,7 @@ Description: Value that will be transformed | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. | minLength | int64 | 1 | false | Specifies the minimum length of the transformed value. -| maxLength | int64 | 20 | false | Specifies the maximum length of the transformed value. +| maxLength | int64 | 100 | false | Specifies the maximum length of the transformed value. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** @@ -387,7 +387,7 @@ Description: Value that will be transformed const newValue = neosync.transformString(value, { preserveLength: false, minLength: 1, - maxLength: 20, + maxLength: 100, seed: 1, }); @@ -413,8 +413,8 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| preserveLength | bool | | true | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| maxLength | int64 | | true | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** @@ -422,8 +422,8 @@ Description: Value that will be transformed ```javascript const newValue = neosync.transformStringPhoneNumber(value, { - preserveLength: false, - maxLength: 1, + preserveLength: false, + maxLength: 100, seed: 1, }); @@ -483,7 +483,7 @@ Generates a 16 digit card number that is valid by Luhn valid by default. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| validLuhn | bool | | true | A boolean indicating whether the generated value should pass the Luhn algorithm check. +| validLuhn | bool | false | false | A boolean indicating whether the generated value should pass the Luhn algorithm check. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -492,7 +492,7 @@ Generates a 16 digit card number that is valid by Luhn valid by default. ```javascript const newValue = neosync.generateCardNumber({ - validLuhn: false, + validLuhn: false, seed: 1, }); @@ -514,7 +514,7 @@ Randomly selects a value from a defined set of categorical values. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| categories | string | | true | A list of comma-separated string values to randomly select from. +| categories | string | "ultimo,proximo,semper" | false | A list of comma-separated string values to randomly select from. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -523,7 +523,7 @@ Randomly selects a value from a defined set of categorical values. ```javascript const newValue = neosync.generateCategorical({ - categories: "", + categories: "ultimo,proximo,semper", seed: 1, }); @@ -545,7 +545,7 @@ Randomly selects a city from a list of predefined US cities. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | | true | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -554,7 +554,7 @@ Randomly selects a city from a list of predefined US cities. ```javascript const newValue = neosync.generateCity({ - maxLength: 1, + maxLength: 100, seed: 1, }); @@ -640,7 +640,7 @@ Generates a random first name between 2 and 12 characters long. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -649,7 +649,7 @@ Generates a random first name between 2 and 12 characters long. ```javascript const newValue = neosync.generateFirstName({ - maxLength: 10000, + maxLength: 100, seed: 1, }); @@ -672,8 +672,8 @@ Generates a random floating point number with a max precision of 17. Go float64 | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | randomizeSign | bool | false | false | A boolean indicating whether the sign of the float should be randomized. -| min | float64 | | true | Specifies the minimum value for the generated float. -| max | float64 | | true | Specifies the maximum value for the generated float +| min | float64 | 1 | false | Specifies the minimum value for the generated float. +| max | float64 | 10000 | false | Specifies the maximum value for the generated float | precision | int64 | | false | An optional parameter that defines the number of significant digits for the generated float. | scale | int64 | | false | An optional parameter that defines the number of decimal places for the generated float. | seed | int64 | | false | An optional seed value used to generate deterministic outputs. @@ -685,8 +685,8 @@ Generates a random floating point number with a max precision of 17. Go float64 const newValue = neosync.generateFloat64({ randomizeSign: false, - min: 1.12, - max: 1.12, + min: 1, + max: 10000, precision: 1, scale: 1, seed: 1, @@ -710,7 +710,7 @@ Generates a randomly selected real full address that exists in the United States | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | | true | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -719,7 +719,7 @@ Generates a randomly selected real full address that exists in the United States ```javascript const newValue = neosync.generateFullAddress({ - maxLength: 1, + maxLength: 100, seed: 1, }); @@ -741,7 +741,7 @@ Generates a new full name consisting of a first and last name. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -750,7 +750,7 @@ Generates a new full name consisting of a first and last name. ```javascript const newValue = neosync.generateFullName({ - maxLength: 10000, + maxLength: 100, seed: 1, }); @@ -806,8 +806,8 @@ Generates a random int64 value with a default length of 4. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | randomizeSign | bool | false | false | A boolean indicating whether the sign of the float should be randomized. -| min | int64 | | true | Specifies the minimum value for the generated int. -| max | int64 | | true | Specifies the maximum value for the generated int. +| min | int64 | 1 | false | Specifies the minimum value for the generated int. +| max | int64 | 10000 | false | Specifies the maximum value for the generated int. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -817,8 +817,8 @@ Generates a random int64 value with a default length of 4. const newValue = neosync.generateInt64({ randomizeSign: false, - min: 1, - max: 1, + min: 1, + max: 10000, seed: 1, }); @@ -869,8 +869,8 @@ Generates a new random international phone number including the + sign and no hy | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| min | int64 | | true | Specifies the minimum value for the generated phone number. -| max | int64 | | true | Specifies the maximum value for the generated phone number. +| min | int64 | 9 | false | Specifies the minimum value for the generated phone number. +| max | int64 | 15 | false | Specifies the maximum value for the generated phone number. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -879,8 +879,8 @@ Generates a new random international phone number including the + sign and no hy ```javascript const newValue = neosync.generateInternationalPhoneNumber({ - min: 1, - max: 1, + min: 9, + max: 15, seed: 1, }); @@ -902,7 +902,7 @@ Generates a random last name. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -911,7 +911,7 @@ Generates a random last name. ```javascript const newValue = neosync.generateLastName({ - maxLength: 10000, + maxLength: 100, seed: 1, }); @@ -933,8 +933,8 @@ Generates a random string of alphanumeric characters.. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| min | int64 | | true | Specifies the minimum length for the generated string. -| max | int64 | | true | Specifies the maximum length for the generated string. +| min | int64 | 1 | false | Specifies the minimum length for the generated string. +| max | int64 | 100 | false | Specifies the maximum length for the generated string. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -943,8 +943,8 @@ Generates a random string of alphanumeric characters.. ```javascript const newValue = neosync.generateRandomString({ - min: 1, - max: 1, + min: 1, + max: 100, seed: 1, }); @@ -1052,7 +1052,7 @@ Randomly generates a street address. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | | true | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1061,7 +1061,7 @@ Randomly generates a street address. ```javascript const newValue = neosync.generateStreetAddress({ - maxLength: 1, + maxLength: 100, seed: 1, }); @@ -1083,8 +1083,8 @@ Generates a random 10 digit phone number and returns it as a string with no hyph | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| min | int64 | | true | Specifies the minimum length for the generated phone number. -| max | int64 | | true | Specifies the maximum length for the generated phone number. +| min | int64 | 9 | false | Specifies the minimum length for the generated phone number. +| max | int64 | 15 | false | Specifies the maximum length for the generated phone number. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1093,8 +1093,8 @@ Generates a random 10 digit phone number and returns it as a string with no hyph ```javascript const newValue = neosync.generateStringPhoneNumber({ - min: 1, - max: 1, + min: 9, + max: 15, seed: 1, }); @@ -1145,7 +1145,7 @@ Randomly generates a username | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 10000 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1154,7 +1154,7 @@ Randomly generates a username ```javascript const newValue = neosync.generateUsername({ - maxLength: 10000, + maxLength: 100, seed: 1, }); diff --git a/frontend/apps/web/@types/neosync-transformers.d.ts b/frontend/apps/web/@types/neosync-transformers.d.ts index 8377deef3e..3c2c4ea0a4 100644 --- a/frontend/apps/web/@types/neosync-transformers.d.ts +++ b/frontend/apps/web/@types/neosync-transformers.d.ts @@ -27,7 +27,7 @@ declare namespace neosync { export interface TransformE164PhoneNumberOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ - preserveLength: boolean; + preserveLength?: boolean; /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ @@ -80,9 +80,9 @@ declare namespace neosync { export interface TransformFloat64Options { /** Specifies the minimum value for the range of the float. */ - randomizationRangeMin: number; + randomizationRangeMin?: number; /** Specifies the maximum value for the randomization range of the float. */ - randomizationRangeMax: number; + randomizationRangeMax?: number; /** An optional parameter that defines the number of significant digits for the float. */ precision?: number; /** An optional parameter that defines the number of decimal places for the float. */ @@ -98,7 +98,7 @@ declare namespace neosync { export interface TransformFullNameOptions { - /** */ + /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ preserveLength?: boolean; @@ -114,9 +114,9 @@ declare namespace neosync { export interface TransformInt64Options { /** Specifies the minimum value for the range of the int. */ - randomizationRangeMin: number; + randomizationRangeMin?: number; /** Specifies the maximum value for the range of the int. */ - randomizationRangeMax: number; + randomizationRangeMax?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -129,7 +129,7 @@ declare namespace neosync { export interface TransformInt64PhoneNumberOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ - preserveLength: boolean; + preserveLength?: boolean; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -174,9 +174,9 @@ declare namespace neosync { export interface TransformStringPhoneNumberOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ - preserveLength: boolean; + preserveLength?: boolean; /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ - maxLength: number; + maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -206,7 +206,7 @@ declare namespace neosync { export interface GenerateCardNumberOptions { /** A boolean indicating whether the generated value should pass the Luhn algorithm check. */ - validLuhn: boolean; + validLuhn?: boolean; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -219,7 +219,7 @@ declare namespace neosync { export interface GenerateCategoricalOptions { /** A list of comma-separated string values to randomly select from. */ - categories: string; + categories?: string; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -232,7 +232,7 @@ declare namespace neosync { export interface GenerateCityOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ - maxLength: number; + maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -288,9 +288,9 @@ declare namespace neosync { /** A boolean indicating whether the sign of the float should be randomized. */ randomizeSign?: boolean; /** Specifies the minimum value for the generated float. */ - min: number; + min?: number; /** Specifies the maximum value for the generated float */ - max: number; + max?: number; /** An optional parameter that defines the number of significant digits for the generated float. */ precision?: number; /** An optional parameter that defines the number of decimal places for the generated float. */ @@ -307,7 +307,7 @@ declare namespace neosync { export interface GenerateFullAddressOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ - maxLength: number; + maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -350,9 +350,9 @@ declare namespace neosync { /** A boolean indicating whether the sign of the float should be randomized. */ randomizeSign?: boolean; /** Specifies the minimum value for the generated int. */ - min: number; + min?: number; /** Specifies the maximum value for the generated int. */ - max: number; + max?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -376,9 +376,9 @@ declare namespace neosync { export interface GenerateInternationalPhoneNumberOptions { /** Specifies the minimum value for the generated phone number. */ - min: number; + min?: number; /** Specifies the maximum value for the generated phone number. */ - max: number; + max?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -404,9 +404,9 @@ declare namespace neosync { export interface GenerateRandomStringOptions { /** Specifies the minimum length for the generated string. */ - min: number; + min?: number; /** Specifies the maximum length for the generated string. */ - max: number; + max?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -452,7 +452,7 @@ declare namespace neosync { export interface GenerateStreetAddressOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ - maxLength: number; + maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } @@ -465,9 +465,9 @@ declare namespace neosync { export interface GenerateStringPhoneNumberOptions { /** Specifies the minimum length for the generated phone number. */ - min: number; + min?: number; /** Specifies the maximum length for the generated phone number. */ - max: number; + max?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index 2bdc7d61c1..08a1a455f0 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateBool struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index 84e42adf5a..f07b285bd5 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateCardNumber struct{} @@ -25,9 +25,14 @@ func NewGenerateCardNumber() *GenerateCardNumber { } func NewGenerateCardNumberOpts( - validLuhn bool, + validLuhnArg *bool, seedArg *int64, ) (*GenerateCardNumberOpts, error) { + validLuhn := bool(false) + if validLuhnArg != nil && !transformer_utils.IsZeroValue(*validLuhnArg) { + validLuhn = *validLuhnArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *GenerateCardNumber) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateCardNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateCardNumberOpts{} - if _, ok := opts["validLuhn"].(bool); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateCardNumber", "validLuhn") + validLuhn, ok := opts["validLuhn"].(bool) + if !ok { + validLuhn = false } - validLuhn := opts["validLuhn"].(bool) transformerOpts.validLuhn = validLuhn var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index 590113b47a..7f4929cc17 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateCategorical struct{} @@ -25,9 +25,14 @@ func NewGenerateCategorical() *GenerateCategorical { } func NewGenerateCategoricalOpts( - categories string, + categoriesArg *string, seedArg *int64, ) (*GenerateCategoricalOpts, error) { + categories := string("ultimo,proximo,semper") + if categoriesArg != nil && !transformer_utils.IsZeroValue(*categoriesArg) { + categories = *categoriesArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *GenerateCategorical) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateCategorical) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateCategoricalOpts{} - if _, ok := opts["categories"].(string); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateCategorical", "categories") + categories, ok := opts["categories"].(string) + if !ok { + categories = "ultimo,proximo,semper" } - categories := opts["categories"].(string) transformerOpts.categories = categories var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index 76d707a263..019c5e8f36 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateCity struct{} @@ -25,9 +25,14 @@ func NewGenerateCity() *GenerateCity { } func NewGenerateCityOpts( - maxLength int64, + maxLengthArg *int64, seedArg *int64, ) (*GenerateCityOpts, error) { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + maxLength = *maxLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *GenerateCity) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateCity) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateCityOpts{} - if _, ok := opts["maxLength"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateCity", "maxLength") + maxLength, ok := opts["maxLength"].(int64) + if !ok { + maxLength = 100 } - maxLength := opts["maxLength"].(int64) transformerOpts.maxLength = maxLength var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index 4abd4caf9a..e9e57f0a85 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateCountry struct{} @@ -29,7 +29,7 @@ func NewGenerateCountryOpts( seedArg *int64, ) (*GenerateCountryOpts, error) { generateFullName := bool(false) - if generateFullNameArg != nil { + if generateFullNameArg != nil && !transformer_utils.IsZeroValue(*generateFullNameArg) { generateFullName = *generateFullNameArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index cf6debb37d..b345233eb1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateEmail struct{} @@ -31,12 +31,12 @@ func NewGenerateEmailOpts( seedArg *int64, ) (*GenerateEmailOpts, error) { maxLength := int64(100000) - if maxLengthArg != nil { + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } emailType := string(GenerateEmailType_UuidV4.String()) - if emailTypeArg != nil { + if emailTypeArg != nil && !transformer_utils.IsZeroValue(*emailTypeArg) { emailType = *emailTypeArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index eeedfd934c..93f93a123d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateFirstName struct{} @@ -28,8 +28,8 @@ func NewGenerateFirstNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateFirstNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -57,7 +57,7 @@ func (t *GenerateFirstName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index 45af37875b..8976ee6273 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateFloat64 struct{} @@ -30,17 +30,27 @@ func NewGenerateFloat64() *GenerateFloat64 { func NewGenerateFloat64Opts( randomizeSignArg *bool, - min float64, - max float64, + minArg *float64, + maxArg *float64, precision *int64, scale *int64, seedArg *int64, ) (*GenerateFloat64Opts, error) { randomizeSign := bool(false) - if randomizeSignArg != nil { + if randomizeSignArg != nil && !transformer_utils.IsZeroValue(*randomizeSignArg) { randomizeSign = *randomizeSignArg } + min := float64(1) + if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + min = *minArg + } + + max := float64(10000) + if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + max = *maxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -73,16 +83,16 @@ func (t *GenerateFloat64) ParseOptions(opts map[string]any) (any, error) { } transformerOpts.randomizeSign = randomizeSign - if _, ok := opts["min"].(float64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateFloat64", "min") + min, ok := opts["min"].(float64) + if !ok { + min = 1 } - min := opts["min"].(float64) transformerOpts.min = min - if _, ok := opts["max"].(float64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateFloat64", "max") + max, ok := opts["max"].(float64) + if !ok { + max = 10000 } - max := opts["max"].(float64) transformerOpts.max = max var precision *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index 67cf3833fa..966fb1cf11 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateFullAddress struct{} @@ -25,9 +25,14 @@ func NewGenerateFullAddress() *GenerateFullAddress { } func NewGenerateFullAddressOpts( - maxLength int64, + maxLengthArg *int64, seedArg *int64, ) (*GenerateFullAddressOpts, error) { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + maxLength = *maxLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *GenerateFullAddress) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateFullAddress) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateFullAddressOpts{} - if _, ok := opts["maxLength"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateFullAddress", "maxLength") + maxLength, ok := opts["maxLength"].(int64) + if !ok { + maxLength = 100 } - maxLength := opts["maxLength"].(int64) transformerOpts.maxLength = maxLength var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index f0eee8b1e8..20b290456b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateFullName struct{} @@ -28,8 +28,8 @@ func NewGenerateFullNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateFullNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -57,7 +57,7 @@ func (t *GenerateFullName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index d46ca6d5f7..ba2ca85e48 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateGender struct{} @@ -31,12 +31,12 @@ func NewGenerateGenderOpts( seedArg *int64, ) (*GenerateGenderOpts, error) { abbreviate := bool(false) - if abbreviateArg != nil { + if abbreviateArg != nil && !transformer_utils.IsZeroValue(*abbreviateArg) { abbreviate = *abbreviateArg } maxLength := int64(10000) - if maxLengthArg != nil { + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index 9f273d716f..ef66fdf15a 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateInt64 struct{} @@ -28,15 +28,25 @@ func NewGenerateInt64() *GenerateInt64 { func NewGenerateInt64Opts( randomizeSignArg *bool, - min int64, - max int64, + minArg *int64, + maxArg *int64, seedArg *int64, ) (*GenerateInt64Opts, error) { randomizeSign := bool(false) - if randomizeSignArg != nil { + if randomizeSignArg != nil && !transformer_utils.IsZeroValue(*randomizeSignArg) { randomizeSign = *randomizeSignArg } + min := int64(1) + if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + min = *minArg + } + + max := int64(10000) + if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + max = *maxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -67,16 +77,16 @@ func (t *GenerateInt64) ParseOptions(opts map[string]any) (any, error) { } transformerOpts.randomizeSign = randomizeSign - if _, ok := opts["min"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateInt64", "min") + min, ok := opts["min"].(int64) + if !ok { + min = 1 } - min := opts["min"].(int64) transformerOpts.min = min - if _, ok := opts["max"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateInt64", "max") + max, ok := opts["max"].(int64) + if !ok { + max = 10000 } - max := opts["max"].(int64) transformerOpts.max = max var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index 4b1cda478e..342e058004 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateInt64PhoneNumber struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index 93771ff975..7538e03f4d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateInternationalPhoneNumber struct{} @@ -26,10 +26,20 @@ func NewGenerateInternationalPhoneNumber() *GenerateInternationalPhoneNumber { } func NewGenerateInternationalPhoneNumberOpts( - min int64, - max int64, + minArg *int64, + maxArg *int64, seedArg *int64, ) (*GenerateInternationalPhoneNumberOpts, error) { + min := int64(9) + if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + min = *minArg + } + + max := int64(15) + if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + max = *maxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,16 +63,16 @@ func (t *GenerateInternationalPhoneNumber) GetJsTemplateData() (*TemplateData, e func (t *GenerateInternationalPhoneNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateInternationalPhoneNumberOpts{} - if _, ok := opts["min"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateInternationalPhoneNumber", "min") + min, ok := opts["min"].(int64) + if !ok { + min = 9 } - min := opts["min"].(int64) transformerOpts.min = min - if _, ok := opts["max"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateInternationalPhoneNumber", "max") + max, ok := opts["max"].(int64) + if !ok { + max = 15 } - max := opts["max"].(int64) transformerOpts.max = max var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index b14e945bf7..e29da788ce 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateLastName struct{} @@ -28,8 +28,8 @@ func NewGenerateLastNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateLastNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -57,7 +57,7 @@ func (t *GenerateLastName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index 7e17bc5088..6c3ea79150 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateRandomString struct{} @@ -26,10 +26,20 @@ func NewGenerateRandomString() *GenerateRandomString { } func NewGenerateRandomStringOpts( - min int64, - max int64, + minArg *int64, + maxArg *int64, seedArg *int64, ) (*GenerateRandomStringOpts, error) { + min := int64(1) + if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + min = *minArg + } + + max := int64(100) + if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + max = *maxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,16 +63,16 @@ func (t *GenerateRandomString) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateRandomString) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateRandomStringOpts{} - if _, ok := opts["min"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateRandomString", "min") + min, ok := opts["min"].(int64) + if !ok { + min = 1 } - min := opts["min"].(int64) transformerOpts.min = min - if _, ok := opts["max"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateRandomString", "max") + max, ok := opts["max"].(int64) + if !ok { + max = 100 } - max := opts["max"].(int64) transformerOpts.max = max var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index 444c4dd79a..29826543d1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateSSN struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index 59d29160a1..8d91bb6365 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateState struct{} @@ -29,7 +29,7 @@ func NewGenerateStateOpts( seedArg *int64, ) (*GenerateStateOpts, error) { generateFullName := bool(false) - if generateFullNameArg != nil { + if generateFullNameArg != nil && !transformer_utils.IsZeroValue(*generateFullNameArg) { generateFullName = *generateFullNameArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index 4cf7647693..c2673f117d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateStreetAddress struct{} @@ -25,9 +25,14 @@ func NewGenerateStreetAddress() *GenerateStreetAddress { } func NewGenerateStreetAddressOpts( - maxLength int64, + maxLengthArg *int64, seedArg *int64, ) (*GenerateStreetAddressOpts, error) { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + maxLength = *maxLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *GenerateStreetAddress) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateStreetAddress) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateStreetAddressOpts{} - if _, ok := opts["maxLength"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateStreetAddress", "maxLength") + maxLength, ok := opts["maxLength"].(int64) + if !ok { + maxLength = 100 } - maxLength := opts["maxLength"].(int64) transformerOpts.maxLength = maxLength var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index 0070207924..9508e1f5de 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateStringPhoneNumber struct{} @@ -26,10 +26,20 @@ func NewGenerateStringPhoneNumber() *GenerateStringPhoneNumber { } func NewGenerateStringPhoneNumberOpts( - min int64, - max int64, + minArg *int64, + maxArg *int64, seedArg *int64, ) (*GenerateStringPhoneNumberOpts, error) { + min := int64(9) + if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + min = *minArg + } + + max := int64(15) + if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + max = *maxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,16 +63,16 @@ func (t *GenerateStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) { func (t *GenerateStringPhoneNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &GenerateStringPhoneNumberOpts{} - if _, ok := opts["min"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateStringPhoneNumber", "min") + min, ok := opts["min"].(int64) + if !ok { + min = 9 } - min := opts["min"].(int64) transformerOpts.min = min - if _, ok := opts["max"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "generateStringPhoneNumber", "max") + max, ok := opts["max"].(int64) + if !ok { + max = 15 } - max := opts["max"].(int64) transformerOpts.max = max var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index f95717afe5..dac7de62fb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateUnixTimestamp struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index 413305d27c..b8879ef7a8 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateUsername struct{} @@ -28,8 +28,8 @@ func NewGenerateUsernameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateUsernameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -57,7 +57,7 @@ func (t *GenerateUsername) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index cbd99cdb08..a0e7d518b9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateUTCTimestamp struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_uuid.go b/worker/pkg/benthos/transformers/gen_generate_uuid.go index 3c6801badb..3b35d2bd6f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_uuid.go +++ b/worker/pkg/benthos/transformers/gen_generate_uuid.go @@ -5,6 +5,7 @@ package transformers import ( + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateUUID struct{} @@ -21,7 +22,7 @@ func NewGenerateUUIDOpts( includeHyphensArg *bool, ) (*GenerateUUIDOpts, error) { includeHyphens := bool(true) - if includeHyphensArg != nil { + if includeHyphensArg != nil && !transformer_utils.IsZeroValue(*includeHyphensArg) { includeHyphens = *includeHyphensArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index 2e1ab3bfb9..cc324cf342 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateZipcode struct{} diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 5228484eb2..66e1547a2e 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformCharacterScramble struct{} diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index 788c5d5b96..7f6bcc4f58 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformE164PhoneNumber struct{} @@ -26,10 +26,15 @@ func NewTransformE164PhoneNumber() *TransformE164PhoneNumber { } func NewTransformE164PhoneNumberOpts( - preserveLength bool, + preserveLengthArg *bool, maxLength *int64, seedArg *int64, ) (*TransformE164PhoneNumberOpts, error) { + preserveLength := bool(false) + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + preserveLength = *preserveLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,10 +58,10 @@ func (t *TransformE164PhoneNumber) GetJsTemplateData() (*TemplateData, error) { func (t *TransformE164PhoneNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &TransformE164PhoneNumberOpts{} - if _, ok := opts["preserveLength"].(bool); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformE164PhoneNumber", "preserveLength") + preserveLength, ok := opts["preserveLength"].(bool) + if !ok { + preserveLength = false } - preserveLength := opts["preserveLength"].(bool) transformerOpts.preserveLength = preserveLength var maxLength *int64 diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index 2e42bb0c7e..f0c04720b5 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformEmail struct{} @@ -39,22 +39,22 @@ func NewTransformEmailOpts( invalidEmailActionArg *string, ) (*TransformEmailOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil { + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg } preserveDomain := bool(false) - if preserveDomainArg != nil { + if preserveDomainArg != nil && !transformer_utils.IsZeroValue(*preserveDomainArg) { preserveDomain = *preserveDomainArg } excludedDomains := any([]any{}) - if excludedDomainsArg != nil { + if excludedDomainsArg != nil && !transformer_utils.IsZeroValue(*excludedDomainsArg) { excludedDomains = *excludedDomainsArg } - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -64,12 +64,12 @@ func NewTransformEmailOpts( } emailType := string(GenerateEmailType_UuidV4.String()) - if emailTypeArg != nil { + if emailTypeArg != nil && !transformer_utils.IsZeroValue(*emailTypeArg) { emailType = *emailTypeArg } invalidEmailAction := string(InvalidEmailAction_Reject.String()) - if invalidEmailActionArg != nil { + if invalidEmailActionArg != nil && !transformer_utils.IsZeroValue(*invalidEmailActionArg) { invalidEmailAction = *invalidEmailActionArg } @@ -115,7 +115,7 @@ func (t *TransformEmail) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index d501318f54..4e613ccea3 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformFirstName struct{} @@ -30,13 +30,13 @@ func NewTransformFirstNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformFirstNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil { + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg } @@ -65,7 +65,7 @@ func (t *TransformFirstName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index e3df94d15a..a29d4383c2 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformFloat64 struct{} @@ -28,12 +28,22 @@ func NewTransformFloat64() *TransformFloat64 { } func NewTransformFloat64Opts( - randomizationRangeMin float64, - randomizationRangeMax float64, + randomizationRangeMinArg *float64, + randomizationRangeMaxArg *float64, precision *int64, scale *int64, seedArg *int64, ) (*TransformFloat64Opts, error) { + randomizationRangeMin := float64(1) + if randomizationRangeMinArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMinArg) { + randomizationRangeMin = *randomizationRangeMinArg + } + + randomizationRangeMax := float64(10000) + if randomizationRangeMaxArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMaxArg) { + randomizationRangeMax = *randomizationRangeMaxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -59,16 +69,16 @@ func (t *TransformFloat64) GetJsTemplateData() (*TemplateData, error) { func (t *TransformFloat64) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &TransformFloat64Opts{} - if _, ok := opts["randomizationRangeMin"].(float64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformFloat64", "randomizationRangeMin") + randomizationRangeMin, ok := opts["randomizationRangeMin"].(float64) + if !ok { + randomizationRangeMin = 1 } - randomizationRangeMin := opts["randomizationRangeMin"].(float64) transformerOpts.randomizationRangeMin = randomizationRangeMin - if _, ok := opts["randomizationRangeMax"].(float64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformFloat64", "randomizationRangeMax") + randomizationRangeMax, ok := opts["randomizationRangeMax"].(float64) + if !ok { + randomizationRangeMax = 10000 } - randomizationRangeMax := opts["randomizationRangeMax"].(float64) transformerOpts.randomizationRangeMax = randomizationRangeMax var precision *int64 diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index 1bcf18852f..e6a7a50deb 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformFullName struct{} @@ -30,13 +30,13 @@ func NewTransformFullNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformFullNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil { + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg } @@ -65,7 +65,7 @@ func (t *TransformFullName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index ce6cfe3f8a..ae1e15845a 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformInt64 struct{} @@ -26,10 +26,20 @@ func NewTransformInt64() *TransformInt64 { } func NewTransformInt64Opts( - randomizationRangeMin int64, - randomizationRangeMax int64, + randomizationRangeMinArg *int64, + randomizationRangeMaxArg *int64, seedArg *int64, ) (*TransformInt64Opts, error) { + randomizationRangeMin := int64(1) + if randomizationRangeMinArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMinArg) { + randomizationRangeMin = *randomizationRangeMinArg + } + + randomizationRangeMax := int64(10000) + if randomizationRangeMaxArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMaxArg) { + randomizationRangeMax = *randomizationRangeMaxArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,16 +63,16 @@ func (t *TransformInt64) GetJsTemplateData() (*TemplateData, error) { func (t *TransformInt64) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &TransformInt64Opts{} - if _, ok := opts["randomizationRangeMin"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformInt64", "randomizationRangeMin") + randomizationRangeMin, ok := opts["randomizationRangeMin"].(int64) + if !ok { + randomizationRangeMin = 1 } - randomizationRangeMin := opts["randomizationRangeMin"].(int64) transformerOpts.randomizationRangeMin = randomizationRangeMin - if _, ok := opts["randomizationRangeMax"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformInt64", "randomizationRangeMax") + randomizationRangeMax, ok := opts["randomizationRangeMax"].(int64) + if !ok { + randomizationRangeMax = 10000 } - randomizationRangeMax := opts["randomizationRangeMax"].(int64) transformerOpts.randomizationRangeMax = randomizationRangeMax var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index c6a8b5cb73..923fcc3168 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformInt64PhoneNumber struct{} @@ -25,9 +25,14 @@ func NewTransformInt64PhoneNumber() *TransformInt64PhoneNumber { } func NewTransformInt64PhoneNumberOpts( - preserveLength bool, + preserveLengthArg *bool, seedArg *int64, ) (*TransformInt64PhoneNumberOpts, error) { + preserveLength := bool(false) + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + preserveLength = *preserveLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -50,10 +55,10 @@ func (t *TransformInt64PhoneNumber) GetJsTemplateData() (*TemplateData, error) { func (t *TransformInt64PhoneNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &TransformInt64PhoneNumberOpts{} - if _, ok := opts["preserveLength"].(bool); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformInt64PhoneNumber", "preserveLength") + preserveLength, ok := opts["preserveLength"].(bool) + if !ok { + preserveLength = false } - preserveLength := opts["preserveLength"].(bool) transformerOpts.preserveLength = preserveLength var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index f9db4a28ed..d78d634863 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformLastName struct{} @@ -30,13 +30,13 @@ func NewTransformLastNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformLastNameOpts, error) { - maxLength := int64(10000) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil { + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg } @@ -65,7 +65,7 @@ func (t *TransformLastName) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index 6e0b8e4e1c..49d1567b7d 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformString struct{} @@ -33,17 +33,17 @@ func NewTransformStringOpts( seedArg *int64, ) (*TransformStringOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil { + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg } minLength := int64(1) - if minLengthArg != nil { + if minLengthArg != nil && !transformer_utils.IsZeroValue(*minLengthArg) { minLength = *minLengthArg } - maxLength := int64(20) - if maxLengthArg != nil { + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -85,7 +85,7 @@ func (t *TransformString) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 20 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 92f77f2898..fbcd49d269 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -7,9 +7,9 @@ package transformers import ( "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformStringPhoneNumber struct{} @@ -26,10 +26,20 @@ func NewTransformStringPhoneNumber() *TransformStringPhoneNumber { } func NewTransformStringPhoneNumberOpts( - preserveLength bool, - maxLength int64, + preserveLengthArg *bool, + maxLengthArg *int64, seedArg *int64, ) (*TransformStringPhoneNumberOpts, error) { + preserveLength := bool(false) + if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + preserveLength = *preserveLengthArg + } + + maxLength := int64(100) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + maxLength = *maxLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -53,16 +63,16 @@ func (t *TransformStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) func (t *TransformStringPhoneNumber) ParseOptions(opts map[string]any) (any, error) { transformerOpts := &TransformStringPhoneNumberOpts{} - if _, ok := opts["preserveLength"].(bool); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformStringPhoneNumber", "preserveLength") + preserveLength, ok := opts["preserveLength"].(bool) + if !ok { + preserveLength = false } - preserveLength := opts["preserveLength"].(bool) transformerOpts.preserveLength = preserveLength - if _, ok := opts["maxLength"].(int64); !ok { - return nil, fmt.Errorf("missing required argument. function: %s argument: %s", "transformStringPhoneNumber", "maxLength") + maxLength, ok := opts["maxLength"].(int64) + if !ok { + maxLength = 100 } - maxLength := opts["maxLength"].(int64) transformerOpts.maxLength = maxLength var seedArg *int64 diff --git a/worker/pkg/benthos/transformers/generate_categorical.go b/worker/pkg/benthos/transformers/generate_categorical.go index 339cd5fcd8..3a6d01e508 100644 --- a/worker/pkg/benthos/transformers/generate_categorical.go +++ b/worker/pkg/benthos/transformers/generate_categorical.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Randomly selects a value from a defined set of categorical values."). - Param(bloblang.NewStringParam("categories").Description("A list of comma-separated string values to randomly select from.")). + Param(bloblang.NewStringParam("categories").Default("ultimo,proximo,semper").Description("A list of comma-separated string values to randomly select from.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_categorical", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_gender_test.go b/worker/pkg/benthos/transformers/generate_gender_test.go index 74c7735b61..5425ec9bb6 100644 --- a/worker/pkg/benthos/transformers/generate_gender_test.go +++ b/worker/pkg/benthos/transformers/generate_gender_test.go @@ -42,3 +42,13 @@ func Test_GenderTransformer(t *testing.T) { assert.Contains(t, valid, res, "Gender should be one of female, male, undefined, nonbinary") assert.Equal(t, int64(len(res.(string))), int64(1), "the length should be 1") } + +func Test_GenderTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_gender()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the gender transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_int64.go b/worker/pkg/benthos/transformers/generate_int64.go index 7b3943b1b5..6603e68988 100644 --- a/worker/pkg/benthos/transformers/generate_int64.go +++ b/worker/pkg/benthos/transformers/generate_int64.go @@ -30,8 +30,8 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random int64 value with a default length of 4."). Param(bloblang.NewBoolParam("randomize_sign").Default(false).Description("A boolean indicating whether the sign of the float should be randomized.")). - Param(bloblang.NewInt64Param("min").Description("Specifies the minimum value for the generated int.")). - Param(bloblang.NewInt64Param("max").Description("Specifies the maximum value for the generated int.")). + Param(bloblang.NewInt64Param("min").Default(1).Description("Specifies the minimum value for the generated int.")). + Param(bloblang.NewInt64Param("max").Default(10000).Description("Specifies the maximum value for the generated int.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_int64", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_int64_test.go b/worker/pkg/benthos/transformers/generate_int64_test.go index 0675811e79..192cf7922e 100644 --- a/worker/pkg/benthos/transformers/generate_int64_test.go +++ b/worker/pkg/benthos/transformers/generate_int64_test.go @@ -82,3 +82,13 @@ func Test_GenerateRandomInt_Benthos(t *testing.T) { assert.GreaterOrEqual(t, res, minValue, "The result should be greater or equal to the minimum") assert.LessOrEqual(t, res, maxValue, "The result should be less or equal to the maximum") } + +func Test_GenerateRandomInt_Benthos_NoOptions(t *testing.T) { + mapping := `root = generate_int64()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the random int transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_international_phone_number.go b/worker/pkg/benthos/transformers/generate_international_phone_number.go index aba30b98a2..715f48cc17 100644 --- a/worker/pkg/benthos/transformers/generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_international_phone_number.go @@ -15,8 +15,8 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a new random international phone number including the + sign and no hyphens."). - Param(bloblang.NewInt64Param("min").Description("Specifies the minimum value for the generated phone number.")). - Param(bloblang.NewInt64Param("max").Description("Specifies the maximum value for the generated phone number.")). + Param(bloblang.NewInt64Param("min").Default(9).Description("Specifies the minimum value for the generated phone number.")). + Param(bloblang.NewInt64Param("max").Default(15).Description("Specifies the maximum value for the generated phone number.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_e164_phone_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_international_phone_number_test.go b/worker/pkg/benthos/transformers/generate_international_phone_number_test.go index b4e76d5e5e..67de222bda 100644 --- a/worker/pkg/benthos/transformers/generate_international_phone_number_test.go +++ b/worker/pkg/benthos/transformers/generate_international_phone_number_test.go @@ -48,6 +48,16 @@ func Test_GenerateInternationalPhoneNumberTransformer(t *testing.T) { assert.LessOrEqual(t, len(res.(string)), 15+1, "Should be less than 16 characters in length. 15 for the number and 1 for the plus sign.") } +func Test_GenerateInternationalPhoneNumberTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_e164_phone_number()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the international phone number transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} + func Test_ValidateE164True(t *testing.T) { val := "+6272636472" diff --git a/worker/pkg/benthos/transformers/generate_last_name.go b/worker/pkg/benthos/transformers/generate_last_name.go index 68b8c31327..749d7536dd 100644 --- a/worker/pkg/benthos/transformers/generate_last_name.go +++ b/worker/pkg/benthos/transformers/generate_last_name.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random last name."). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_last_name", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_last_name_test.go b/worker/pkg/benthos/transformers/generate_last_name_test.go index 25421cbb1c..384c345889 100644 --- a/worker/pkg/benthos/transformers/generate_last_name_test.go +++ b/worker/pkg/benthos/transformers/generate_last_name_test.go @@ -56,3 +56,14 @@ func Test_GenerateRandomLastNameTransformer(t *testing.T) { assert.NotEmpty(t, resStr) assert.LessOrEqual(t, int64(len(resStr)), maxCharacterLimit, "The last name should be less than or equal to max char limit") } + +func Test_GenerateRandomLastNameTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_last_name()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the last name transformer") + assert.NotNil(t, ex) + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_random_string.go index 89d462ea00..69b1bdad36 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -13,8 +13,8 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random string of alphanumeric characters.."). - Param(bloblang.NewInt64Param("min").Description("Specifies the minimum length for the generated string.")). - Param(bloblang.NewInt64Param("max").Description("Specifies the maximum length for the generated string.")). + Param(bloblang.NewInt64Param("min").Default(1).Description("Specifies the minimum length for the generated string.")). + Param(bloblang.NewInt64Param("max").Default(100).Description("Specifies the maximum length for the generated string.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_string", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_random_string_test.go b/worker/pkg/benthos/transformers/generate_random_string_test.go index c1957519df..831a6a20dd 100644 --- a/worker/pkg/benthos/transformers/generate_random_string_test.go +++ b/worker/pkg/benthos/transformers/generate_random_string_test.go @@ -23,3 +23,13 @@ func Test_GenerateRandomStringTransformerWithValue(t *testing.T) { assert.GreaterOrEqual(t, int64(len(res.(string))), minValue, "The output string should be greater than or equal to the min") assert.LessOrEqual(t, int64(len(res.(string))), maxValue, "The output string should be less than or equal to the max") } + +func Test_GenerateRandomStringTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_string()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the generate random string transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_state_test.go b/worker/pkg/benthos/transformers/generate_state_test.go index 472422beca..bc9788c912 100644 --- a/worker/pkg/benthos/transformers/generate_state_test.go +++ b/worker/pkg/benthos/transformers/generate_state_test.go @@ -81,3 +81,14 @@ func Test_StateTransformer(t *testing.T) { assert.Len(t, res, 2) assert.True(t, stateExists, "The generated state should exist in the states.go file") } + +func Test_StateTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_state()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the state transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) + +} diff --git a/worker/pkg/benthos/transformers/generate_street_address.go b/worker/pkg/benthos/transformers/generate_street_address.go index 6d3f48d38c..9e9070ef0f 100644 --- a/worker/pkg/benthos/transformers/generate_street_address.go +++ b/worker/pkg/benthos/transformers/generate_street_address.go @@ -22,7 +22,7 @@ type Address struct { func init() { spec := bloblang.NewPluginSpec(). Description("Randomly generates a street address."). - Param(bloblang.NewInt64Param("max_length").Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_street_address", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_street_address_test.go b/worker/pkg/benthos/transformers/generate_street_address_test.go index f37934f002..b557ad772a 100644 --- a/worker/pkg/benthos/transformers/generate_street_address_test.go +++ b/worker/pkg/benthos/transformers/generate_street_address_test.go @@ -48,3 +48,13 @@ func Test_StreetAddressTransformer(t *testing.T) { assert.IsType(t, Address{}.Address1, res, "The returned street address should be a string") assert.LessOrEqual(t, int64(len(res.(string))), maxLength, fmt.Sprintf("The city should be less than or equal to the max length. This is the error street address:%s", res)) } + +func Test_StreetAddressTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_street_address()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the street address transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_string_phone_number.go b/worker/pkg/benthos/transformers/generate_string_phone_number.go index a09fe3a69e..afdefeef6d 100644 --- a/worker/pkg/benthos/transformers/generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_string_phone_number.go @@ -13,8 +13,8 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Generates a random 10 digit phone number and returns it as a string with no hyphens."). - Param(bloblang.NewInt64Param("min").Description("Specifies the minimum length for the generated phone number.")). - Param(bloblang.NewInt64Param("max").Description("Specifies the maximum length for the generated phone number.")). + Param(bloblang.NewInt64Param("min").Default(9).Description("Specifies the minimum length for the generated phone number.")). + Param(bloblang.NewInt64Param("max").Default(15).Description("Specifies the maximum length for the generated phone number.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_string_phone_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_string_phone_number_test.go b/worker/pkg/benthos/transformers/generate_string_phone_number_test.go index afc78a49ec..547bc96a1c 100644 --- a/worker/pkg/benthos/transformers/generate_string_phone_number_test.go +++ b/worker/pkg/benthos/transformers/generate_string_phone_number_test.go @@ -57,3 +57,13 @@ func Test_GenerateStringPhoneNumberTransformer(t *testing.T) { assert.GreaterOrEqual(t, len(res.(string)), 8, "Should be greater than 9 characters in length. 9 for the number and 1 for the plus sign.") assert.LessOrEqual(t, len(res.(string)), 15, "Should be less than 16 characters in length. 15 for the number and 1 for the plus sign.") } + +func Test_GenerateStringPhoneNumberTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_string_phone_number()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the phone transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_username.go b/worker/pkg/benthos/transformers/generate_username.go index 30c7a49549..8b0302fcd0 100644 --- a/worker/pkg/benthos/transformers/generate_username.go +++ b/worker/pkg/benthos/transformers/generate_username.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Randomly generates a username"). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_username", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/generate_username_test.go b/worker/pkg/benthos/transformers/generate_username_test.go index fc86c487b5..6998f04042 100644 --- a/worker/pkg/benthos/transformers/generate_username_test.go +++ b/worker/pkg/benthos/transformers/generate_username_test.go @@ -49,3 +49,14 @@ func Test_UsernamelTransformer(t *testing.T) { assert.IsType(t, "", res, "The expected username should have a valid username") } + +func Test_UsernamelTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_username()` + ex, err := bloblang.Parse(mapping) + + assert.NoError(t, err, "failed to parse the realistic username transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/generate_uuid_test.go b/worker/pkg/benthos/transformers/generate_uuid_test.go index e4daa720ad..584a91763b 100644 --- a/worker/pkg/benthos/transformers/generate_uuid_test.go +++ b/worker/pkg/benthos/transformers/generate_uuid_test.go @@ -39,3 +39,13 @@ func TestUUIDTransformer(t *testing.T) { assert.Len(t, res.(string), 36, "UUIDs with hyphens must have 36 characters") } + +func TestUUIDTransformer_NoOptions(t *testing.T) { + mapping := `root = generate_uuid()` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the uuid transformer") + + res, err := ex.Query(nil) // input is ignored here + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 8aa00921a2..b59bd2bfe9 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -100,9 +100,9 @@ import ( "fmt" {{ end }} {{- if eq .HasSeedParam true}} - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" {{ end }} + transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type {{.StructName}} struct{} @@ -149,7 +149,7 @@ func New{{.StructName}}Opts( } {{ else if $param.HasDefault }} {{$param.Name}} := {{$param.TypeStr}}({{$param.Default}}) - if {{$param.Name}}Arg != nil { + if {{$param.Name}}Arg != nil && !transformer_utils.IsZeroValue(*{{$param.Name}}Arg) { {{$param.Name}} = *{{$param.Name}}Arg } {{ end }} diff --git a/worker/pkg/benthos/transformers/transform_e164_phone_number.go b/worker/pkg/benthos/transformers/transform_e164_phone_number.go index a4e94661e9..5c48514779 100644 --- a/worker/pkg/benthos/transformers/transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_e164_phone_number.go @@ -16,7 +16,7 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Transforms an existing E164 formatted phone number."). Param(bloblang.NewAnyParam("value").Optional()). - Param(bloblang.NewBoolParam("preserve_length").Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). + Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("max_length").Optional().Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) diff --git a/worker/pkg/benthos/transformers/transform_e164_phone_number_test.go b/worker/pkg/benthos/transformers/transform_e164_phone_number_test.go index db04d2645d..ef5847652e 100644 --- a/worker/pkg/benthos/transformers/transform_e164_phone_number_test.go +++ b/worker/pkg/benthos/transformers/transform_e164_phone_number_test.go @@ -68,3 +68,14 @@ func Test_TransformE164PhoneTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformE164PhoneTransformer_NoOptions(t *testing.T) { + nilE164Phone := "12323" + mapping := fmt.Sprintf(`root = transform_e164_phone_number(value:%q)`, nilE164Phone) + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the e164 phone transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_email.go b/worker/pkg/benthos/transformers/transform_email.go index 82362482bd..10b87caa13 100644 --- a/worker/pkg/benthos/transformers/transform_email.go +++ b/worker/pkg/benthos/transformers/transform_email.go @@ -42,7 +42,7 @@ func init() { Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewBoolParam("preserve_domain").Default(false).Description("A boolean indicating whether the domain part of the email should be preserved.")). Param(bloblang.NewAnyParam("excluded_domains").Default([]any{}).Description("A list of domains that should be excluded from the transformation")). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used for generating deterministic transformations.")). Param(bloblang.NewStringParam("email_type").Default(GenerateEmailType_UuidV4.String()).Description("Specifies the type of email to transform, with options including `uuidv4`, `fullname`, or `any`.")). Param(bloblang.NewStringParam("invalid_email_action").Default(InvalidEmailAction_Reject.String()).Description("Specifies the action to take when an invalid email is encountered, with options including `reject`, `passthrough`, `null`, or `generate`.")) diff --git a/worker/pkg/benthos/transformers/transform_email_test.go b/worker/pkg/benthos/transformers/transform_email_test.go index 049e04ad30..8831f10fb7 100644 --- a/worker/pkg/benthos/transformers/transform_email_test.go +++ b/worker/pkg/benthos/transformers/transform_email_test.go @@ -395,6 +395,16 @@ func Test_TransformEmailTransformer_InvalidEmailArg(t *testing.T) { require.NoError(t, err) } +func Test_TransformEmailTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_email(value:%q)`, "nick@neosync.dev") + ex, err := bloblang.Parse(mapping) + require.NoError(t, err, "failed to parse the email transformer") + + res, err := ex.Query(nil) + require.NoError(t, err) + require.NotEmpty(t, res) +} + func Test_fromAnyToStringSlice(t *testing.T) { var foo any = []any{"123", "456"} output, err := fromAnyToStringSlice(foo) diff --git a/worker/pkg/benthos/transformers/transform_first_name.go b/worker/pkg/benthos/transformers/transform_first_name.go index 34920fece6..66fa0a6b3d 100644 --- a/worker/pkg/benthos/transformers/transform_first_name.go +++ b/worker/pkg/benthos/transformers/transform_first_name.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Transforms an existing first name"). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewAnyParam("value").Optional()). Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used for generating deterministic transformations.")) diff --git a/worker/pkg/benthos/transformers/transform_first_name_test.go b/worker/pkg/benthos/transformers/transform_first_name_test.go index 4a9667fe0d..e8a0eeeaa1 100644 --- a/worker/pkg/benthos/transformers/transform_first_name_test.go +++ b/worker/pkg/benthos/transformers/transform_first_name_test.go @@ -105,3 +105,13 @@ func Test_TransformFirstNameTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformFirstNameTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_first_name(value:%q)`, "name") + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the first name transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_float.go b/worker/pkg/benthos/transformers/transform_float.go index 46b13f6979..703cb3357c 100644 --- a/worker/pkg/benthos/transformers/transform_float.go +++ b/worker/pkg/benthos/transformers/transform_float.go @@ -17,8 +17,8 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Anonymizes and transforms an existing float value."). Param(bloblang.NewAnyParam("value").Optional()). - Param(bloblang.NewFloat64Param("randomization_range_min").Description("Specifies the minimum value for the range of the float.")). - Param(bloblang.NewFloat64Param("randomization_range_max").Description("Specifies the maximum value for the randomization range of the float.")). + Param(bloblang.NewFloat64Param("randomization_range_min").Default(1).Description("Specifies the minimum value for the range of the float.")). + Param(bloblang.NewFloat64Param("randomization_range_max").Default(10000).Description("Specifies the maximum value for the randomization range of the float.")). Param(bloblang.NewInt64Param("precision").Optional().Description("An optional parameter that defines the number of significant digits for the float.")). Param(bloblang.NewInt64Param("scale").Optional().Description("An optional parameter that defines the number of decimal places for the float.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used for generating deterministic transformations.")) diff --git a/worker/pkg/benthos/transformers/transform_float_test.go b/worker/pkg/benthos/transformers/transform_float_test.go index 91d5006d7a..e9a7f2124c 100644 --- a/worker/pkg/benthos/transformers/transform_float_test.go +++ b/worker/pkg/benthos/transformers/transform_float_test.go @@ -49,6 +49,17 @@ func Test_TransformFloat64_Benthos(t *testing.T) { } } +func Test_TransformFloat64_Benthos_NoOptions(t *testing.T) { + val := float64(27.35) + mapping := fmt.Sprintf(`root = transform_float64(value:%f)`, val) + ex, err := bloblang.Parse(mapping) + require.NoError(t, err, "failed to parse the email transformer") + + res, err := ex.Query(nil) + require.NoError(t, err) + require.NotEmpty(t, res) +} + func Test_calculateMaxNumber(t *testing.T) { tests := []struct { precision int diff --git a/worker/pkg/benthos/transformers/transform_full_name.go b/worker/pkg/benthos/transformers/transform_full_name.go index bfd1bb173f..6c529127e9 100644 --- a/worker/pkg/benthos/transformers/transform_full_name.go +++ b/worker/pkg/benthos/transformers/transform_full_name.go @@ -15,7 +15,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Transforms an existing full name."). - Param(bloblang.NewInt64Param("max_length").Default(10000).Default("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewAnyParam("value").Optional()). Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used for generating deterministic transformations.")) diff --git a/worker/pkg/benthos/transformers/transform_full_name_test.go b/worker/pkg/benthos/transformers/transform_full_name_test.go index 15dc4642d7..fe2e48e0eb 100644 --- a/worker/pkg/benthos/transformers/transform_full_name_test.go +++ b/worker/pkg/benthos/transformers/transform_full_name_test.go @@ -75,3 +75,13 @@ func Test_TransformFullNamelTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformFullNamelTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_full_name(value:%q)`, "full name") + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the full name transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_int64.go b/worker/pkg/benthos/transformers/transform_int64.go index b5db1830c5..ed3dcdbf82 100644 --- a/worker/pkg/benthos/transformers/transform_int64.go +++ b/worker/pkg/benthos/transformers/transform_int64.go @@ -15,8 +15,8 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Anonymizes and transforms an existing int64 value."). Param(bloblang.NewAnyParam("value").Optional()). - Param(bloblang.NewInt64Param("randomization_range_min").Description("Specifies the minimum value for the range of the int.")). - Param(bloblang.NewInt64Param("randomization_range_max").Description("Specifies the maximum value for the range of the int.")). + Param(bloblang.NewInt64Param("randomization_range_min").Default(1).Description("Specifies the minimum value for the range of the int.")). + Param(bloblang.NewInt64Param("randomization_range_max").Default(10000).Description("Specifies the maximum value for the range of the int.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("transform_int64", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/transform_int64_phone_number.go b/worker/pkg/benthos/transformers/transform_int64_phone_number.go index 227d8ba3fc..3897fb250c 100644 --- a/worker/pkg/benthos/transformers/transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_int64_phone_number.go @@ -16,7 +16,7 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Anonymizes and transforms an existing int64 phone number."). Param(bloblang.NewAnyParam("value").Optional()). - Param(bloblang.NewBoolParam("preserve_length").Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). + Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("transform_int64_phone_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/transform_int64_phone_number_test.go b/worker/pkg/benthos/transformers/transform_int64_phone_number_test.go index 4a7338a30b..90c551d908 100644 --- a/worker/pkg/benthos/transformers/transform_int64_phone_number_test.go +++ b/worker/pkg/benthos/transformers/transform_int64_phone_number_test.go @@ -75,3 +75,13 @@ func Test_TransformIntPhoneTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformIntPhoneTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_int64_phone_number(value:%d)`, 123123) + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the int phone number transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_int64_test.go b/worker/pkg/benthos/transformers/transform_int64_test.go index 27d429d343..84425032fe 100644 --- a/worker/pkg/benthos/transformers/transform_int64_test.go +++ b/worker/pkg/benthos/transformers/transform_int64_test.go @@ -47,3 +47,14 @@ func Test_TransformInt64_Benthos(t *testing.T) { require.Error(t, err, "Expected the pointer to resolve to an int64") } } + +func Test_TransformInt64_Benthos_NoOptions(t *testing.T) { + val := int64(27) + mapping := fmt.Sprintf(`root = transform_int64(value:%d)`, val) + ex, err := bloblang.Parse(mapping) + require.NoError(t, err, "failed to parse the email transformer") + + res, err := ex.Query(nil) + require.NoError(t, err) + require.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_lastname.go b/worker/pkg/benthos/transformers/transform_lastname.go index 77f91c859b..105ee56a6d 100644 --- a/worker/pkg/benthos/transformers/transform_lastname.go +++ b/worker/pkg/benthos/transformers/transform_lastname.go @@ -14,7 +14,7 @@ import ( func init() { spec := bloblang.NewPluginSpec(). Description("Anonymizes and transforms an existing last name."). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewAnyParam("value").Optional()). Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used for generating deterministic transformations.")) diff --git a/worker/pkg/benthos/transformers/transform_lastname_test.go b/worker/pkg/benthos/transformers/transform_lastname_test.go index 4297a60dc4..853fdf6dbf 100644 --- a/worker/pkg/benthos/transformers/transform_lastname_test.go +++ b/worker/pkg/benthos/transformers/transform_lastname_test.go @@ -92,3 +92,13 @@ func Test_TransformLastNameTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformLastNameTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_last_name(value:%q)`, "lastname") + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the last name transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_string.go b/worker/pkg/benthos/transformers/transform_string.go index 7ed06b21e4..41234fdd4c 100644 --- a/worker/pkg/benthos/transformers/transform_string.go +++ b/worker/pkg/benthos/transformers/transform_string.go @@ -17,7 +17,7 @@ func init() { Param(bloblang.NewAnyParam("value").Optional()). Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). Param(bloblang.NewInt64Param("min_length").Default(1).Description("Specifies the minimum length of the transformed value.")). - Param(bloblang.NewInt64Param("max_length").Default(20).Description("Specifies the maximum length of the transformed value.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length of the transformed value.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("transform_string", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/transform_string_phone_number.go b/worker/pkg/benthos/transformers/transform_string_phone_number.go index 455f25195a..e587089f12 100644 --- a/worker/pkg/benthos/transformers/transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_string_phone_number.go @@ -15,8 +15,8 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Anonymizes and transforms an existing phone number that is typed as a string."). Param(bloblang.NewAnyParam("value").Optional()). - Param(bloblang.NewBoolParam("preserve_length").Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). - Param(bloblang.NewInt64Param("max_length").Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("transform_phone_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/transform_string_phone_number_test.go b/worker/pkg/benthos/transformers/transform_string_phone_number_test.go index 20f6467eb6..4e2d308cd0 100644 --- a/worker/pkg/benthos/transformers/transform_string_phone_number_test.go +++ b/worker/pkg/benthos/transformers/transform_string_phone_number_test.go @@ -58,3 +58,13 @@ func Test_TransformStringPhoneNumberTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformStringPhoneNumberTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_phone_number(value:%q)`, testPhone) + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the transform phone transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transform_string_test.go b/worker/pkg/benthos/transformers/transform_string_test.go index 38aa9ea75c..c0fa7fab8c 100644 --- a/worker/pkg/benthos/transformers/transform_string_test.go +++ b/worker/pkg/benthos/transformers/transform_string_test.go @@ -68,3 +68,13 @@ func Test_TransformStringTransformerWithEmptyValue(t *testing.T) { _, err = ex.Query(nil) assert.NoError(t, err) } + +func Test_TransformStringTransformer_NoOptions(t *testing.T) { + mapping := fmt.Sprintf(`root = transform_string(value:%q)`, testStringValue) + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the email transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + assert.NotEmpty(t, res) +} diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 96a0eb909f..0abeb80d22 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -8,7 +8,6 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" presidioapi "github.com/nucleuscloud/neosync/internal/ee/presidio" ee_transformer_fns "github.com/nucleuscloud/neosync/internal/ee/transformers/functions" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type TransformerExecutor struct { @@ -46,7 +45,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform opt(execCfg) } - maxLength := int64(10000) // TODO: update this based on colInfo if available + maxLength := int64(100) // TODO: update this based on colInfo if available switch transformerConfig.GetConfig().(type) { case *mgmtv1alpha1.TransformerConfig_PassthroughConfig: return &TransformerExecutor{ @@ -56,11 +55,14 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig: - categories := transformerConfig.GetGenerateCategoricalConfig().GetCategories() - opts, err := NewGenerateCategoricalOpts(categories, nil) + config := transformerConfig.GetGenerateCategoricalConfig() + fmt.Println(config.Categories) + + opts, err := NewGenerateCategoricalOpts(&config.Categories, nil) if err != nil { return nil, err } + fmt.Println(opts.categories) generate := NewGenerateCategorical().Generate return &TransformerExecutor{ Opts: opts, @@ -97,9 +99,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, }, nil case *mgmtv1alpha1.TransformerConfig_TransformInt64Config: - rMin := transformerConfig.GetTransformInt64Config().GetRandomizationRangeMin() - rMax := transformerConfig.GetTransformInt64Config().GetRandomizationRangeMax() - opts, err := NewTransformInt64Opts(rMin, rMax, nil) + config := transformerConfig.GetTransformInt64Config() + opts, err := NewTransformInt64Opts(&config.RandomizationRangeMin, &config.RandomizationRangeMax, nil) if err != nil { return nil, err } @@ -178,8 +179,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig: - luhn := transformerConfig.GetGenerateCardNumberConfig().GetValidLuhn() - opts, err := NewGenerateCardNumberOpts(luhn, nil) + config := transformerConfig.GetGenerateCardNumberConfig() + opts, err := NewGenerateCardNumberOpts(&config.ValidLuhn, nil) if err != nil { return nil, err } @@ -192,7 +193,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCityConfig: - opts, err := NewGenerateCityOpts(maxLength, nil) + opts, err := NewGenerateCityOpts(&maxLength, nil) if err != nil { return nil, err } @@ -205,9 +206,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: - minValue := transformerConfig.GetGenerateE164PhoneNumberConfig().GetMin() - maxValue := transformerConfig.GetGenerateE164PhoneNumberConfig().GetMax() - opts, err := NewGenerateInternationalPhoneNumberOpts(minValue, maxValue, nil) + config := transformerConfig.GetGenerateE164PhoneNumberConfig() + opts, err := NewGenerateInternationalPhoneNumberOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } @@ -235,8 +235,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform config := transformerConfig.GetGenerateFloat64Config() opts, err := NewGenerateFloat64Opts( &config.RandomizeSign, - config.GetMin(), - config.GetMax(), + &config.Min, + &config.Max, &config.Precision, nil, // TODO: update scale based on colInfo if available nil, @@ -253,7 +253,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig: - opts, err := NewGenerateFullAddressOpts(maxLength, nil) + opts, err := NewGenerateFullAddressOpts(&maxLength, nil) if err != nil { return nil, err } @@ -307,7 +307,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateInt64Config: config := transformerConfig.GetGenerateInt64Config() - opts, err := NewGenerateInt64Opts(&config.RandomizeSign, config.GetMin(), config.GetMax(), nil) + opts, err := NewGenerateInt64Opts(&config.RandomizeSign, &config.Min, &config.Max, nil) if err != nil { return nil, err } @@ -373,7 +373,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig: - opts, err := NewGenerateStreetAddressOpts(maxLength, nil) + opts, err := NewGenerateStreetAddressOpts(&maxLength, nil) if err != nil { return nil, err } @@ -386,11 +386,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig: - minValue := transformerConfig.GetGenerateStringPhoneNumberConfig().GetMin() - maxValue := transformerConfig.GetGenerateStringPhoneNumberConfig().GetMax() - minValue = transformer_utils.MinInt(minValue, maxLength) - maxValue = transformer_utils.Ceil(maxValue, maxLength) - opts, err := NewGenerateStringPhoneNumberOpts(minValue, maxValue, nil) + config := transformerConfig.GetGenerateStringPhoneNumberConfig() + opts, err := NewGenerateStringPhoneNumberOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } @@ -404,7 +401,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOpts(config.GetMin(), config.GetMax(), nil) + opts, err := NewGenerateRandomStringOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } @@ -484,7 +481,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig: config := transformerConfig.GetTransformE164PhoneNumberConfig() - opts, err := NewTransformE164PhoneNumberOpts(config.GetPreserveLength(), &maxLength, nil) + opts, err := NewTransformE164PhoneNumberOpts(&config.PreserveLength, &maxLength, nil) if err != nil { return nil, err } @@ -513,8 +510,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFloat64Config: config := transformerConfig.GetTransformFloat64Config() opts, err := NewTransformFloat64Opts( - config.GetRandomizationRangeMin(), - config.GetRandomizationRangeMax(), + &config.RandomizationRangeMin, + &config.RandomizationRangeMax, nil, // TODO: update precision based on colInfo if available nil, // TODO: update scale based on colInfo if available nil, @@ -532,7 +529,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig: config := transformerConfig.GetTransformInt64PhoneNumberConfig() - opts, err := NewTransformInt64PhoneNumberOpts(config.GetPreserveLength(), nil) + opts, err := NewTransformInt64PhoneNumberOpts(&config.PreserveLength, nil) if err != nil { return nil, err } @@ -560,7 +557,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformStringPhoneNumberOpts(config.GetPreserveLength(), maxLength, nil) + opts, err := NewTransformStringPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) if err != nil { return nil, err } diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformers/transformer_initializer_test.go new file mode 100644 index 0000000000..e8460027a5 --- /dev/null +++ b/worker/pkg/benthos/transformers/transformer_initializer_test.go @@ -0,0 +1,152 @@ +package transformers + +import ( + "testing" + + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + "github.com/stretchr/testify/require" +) + +func Test_InitializeTransformerByConfigType(t *testing.T) { + t.Run("PassthroughConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test", nil) + require.NoError(t, err) + require.Equal(t, "test", result) + }) + + t.Run("GenerateCategoricalConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{ + GenerateCategoricalConfig: &mgmtv1alpha1.GenerateCategorical{ + Categories: "A,B,C", + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Contains(t, []string{"A", "B", "C"}, result) + }) + + t.Run("GenerateCategoricalConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{ + GenerateCategoricalConfig: &mgmtv1alpha1.GenerateCategorical{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + + t.Run("GenerateBoolConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateBoolConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, bool(true), result) + }) + + t.Run("TransformStringConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ + TransformStringConfig: &mgmtv1alpha1.TransformString{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test", executor.Opts) + require.NoError(t, err) + require.IsType(t, "test", result) + require.Len(t, result.(string), 4) + }) + + t.Run("TransformInt64Config", func(t *testing.T) { + min, max := int64(1), int64(100) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ + TransformInt64Config: &mgmtv1alpha1.TransformInt64{ + RandomizationRangeMin: min, + RandomizationRangeMax: max, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(int64(50), executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.GreaterOrEqual(t, result.(int64), min) + require.LessOrEqual(t, result.(int64), max) + }) + + t.Run("TransformFullNameConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ + TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("John Doe", executor.Opts) + require.NoError(t, err) + require.IsType(t, "John Doe", result) + require.Len(t, result.(string), 8) + }) + + t.Run("GenerateEmailConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateEmailConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$`, result) + }) + + t.Run("TransformEmailConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ + TransformEmailConfig: &mgmtv1alpha1.TransformEmail{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test@example.com", executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^[a-zA-Z0-9._%+-]+@example\.com$`, result) + require.Len(t, result.(string), 16) + }) + + // Add more subtests for other config types... + + t.Run("UnsupportedConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: nil, + } + _, err := InitializeTransformerByConfigType(config) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported transformer") + }) +} diff --git a/worker/pkg/benthos/transformers/utils/type_utils.go b/worker/pkg/benthos/transformers/utils/type_utils.go new file mode 100644 index 0000000000..2b66194ce0 --- /dev/null +++ b/worker/pkg/benthos/transformers/utils/type_utils.go @@ -0,0 +1,19 @@ +package transformer_utils + +import "reflect" + +func IsZeroValue[T any](value T) bool { + var zero T + + if reflect.DeepEqual(value, zero) { + return true + } + + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.Slice, reflect.Map: + return v.Len() == 0 + } + + return false +} diff --git a/worker/pkg/benthos/transformers/utils/type_utils_test.go b/worker/pkg/benthos/transformers/utils/type_utils_test.go new file mode 100644 index 0000000000..99be36fdc1 --- /dev/null +++ b/worker/pkg/benthos/transformers/utils/type_utils_test.go @@ -0,0 +1,77 @@ +package transformer_utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_IsZeroValue(t *testing.T) { + t.Run("Primitive Types", func(t *testing.T) { + t.Run("Integer", func(t *testing.T) { + require.True(t, IsZeroValue(0)) + require.False(t, IsZeroValue(1)) + require.False(t, IsZeroValue(-1)) + }) + + t.Run("Float", func(t *testing.T) { + require.True(t, IsZeroValue(0.0)) + require.False(t, IsZeroValue(0.1)) + require.False(t, IsZeroValue(-0.1)) + }) + + t.Run("Boolean", func(t *testing.T) { + require.True(t, IsZeroValue(false)) + require.False(t, IsZeroValue(true)) + }) + + t.Run("String", func(t *testing.T) { + require.True(t, IsZeroValue("")) + require.False(t, IsZeroValue("hello")) + }) + }) + + t.Run("Complex Types", func(t *testing.T) { + t.Run("Slice", func(t *testing.T) { + require.True(t, IsZeroValue([]int(nil))) + require.True(t, IsZeroValue([]int{})) + require.False(t, IsZeroValue([]int{1, 2, 3})) + }) + + t.Run("Map", func(t *testing.T) { + require.True(t, IsZeroValue(map[string]int(nil))) + require.True(t, IsZeroValue(map[string]int{})) + require.False(t, IsZeroValue(map[string]int{"a": 1})) + }) + + t.Run("Struct", func(t *testing.T) { + type TestStruct struct { + A int + B string + } + require.True(t, IsZeroValue(TestStruct{})) + require.False(t, IsZeroValue(TestStruct{A: 1})) + require.False(t, IsZeroValue(TestStruct{B: "hello"})) + }) + + t.Run("Pointer", func(t *testing.T) { + var nilPtr *int + value := 5 + ptr := &value + require.True(t, IsZeroValue(nilPtr)) + require.False(t, IsZeroValue(ptr)) + }) + }) + + t.Run("Custom Types", func(t *testing.T) { + type CustomInt int + require.True(t, IsZeroValue(CustomInt(0))) + require.False(t, IsZeroValue(CustomInt(1))) + + type CustomStruct struct { + Value int + } + require.True(t, IsZeroValue(CustomStruct{})) + require.False(t, IsZeroValue(CustomStruct{Value: 1})) + }) +} From 2ff876eb4427d4f6ee6bd423e613b6f9fa21b04d Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 12:06:29 -0700 Subject: [PATCH 03/32] add transfomer initializer tests --- .../gen-javascript-transformer.md | 8 +- .../benthos/transformers/gen_generate_bool.go | 1 - .../transformers/gen_generate_card_number.go | 1 - .../transformers/gen_generate_categorical.go | 1 - .../benthos/transformers/gen_generate_city.go | 1 - .../transformers/gen_generate_country.go | 1 - .../transformers/gen_generate_email.go | 1 - .../transformers/gen_generate_first_name.go | 1 - .../transformers/gen_generate_float.go | 1 - .../transformers/gen_generate_full_address.go | 1 - .../transformers/gen_generate_full_name.go | 1 - .../transformers/gen_generate_gender.go | 5 +- .../transformers/gen_generate_int64.go | 1 - .../gen_generate_int64_phone_number.go | 1 - ...gen_generate_international_phone_number.go | 1 - .../transformers/gen_generate_last_name.go | 1 - .../gen_generate_random_string.go | 1 - .../benthos/transformers/gen_generate_ssn.go | 1 - .../transformers/gen_generate_state.go | 1 - .../gen_generate_street_address.go | 1 - .../gen_generate_string_phone_number.go | 1 - .../gen_generate_unix_timestamp.go | 1 - .../transformers/gen_generate_username.go | 1 - .../gen_generate_utc_timestamp.go | 1 - .../benthos/transformers/gen_generate_uuid.go | 3 +- .../transformers/gen_generate_zipcode.go | 1 - .../gen_transform_character_scramble.go | 1 - .../gen_transform_e164_phone_number.go | 16 +- .../transformers/gen_transform_email.go | 1 - .../transformers/gen_transform_first_name.go | 1 - .../transformers/gen_transform_float.go | 1 - .../transformers/gen_transform_full_name.go | 1 - .../transformers/gen_transform_int64.go | 1 - .../gen_transform_int64_phone_number.go | 1 - .../transformers/gen_transform_lastname.go | 1 - .../transformers/gen_transform_string.go | 1 - .../gen_transform_string_phone_number.go | 1 - .../benthos/transformers/generate_gender.go | 2 +- .../neosync_transformer_generator.go | 2 + .../transform_e164_phone_number.go | 4 +- .../transformers/transformer_initializer.go | 43 +- .../transformer_initializer_test.go | 964 +++++++++++++++++- 42 files changed, 1004 insertions(+), 76 deletions(-) diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index 00b87ecb9a..fb6aea2703 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -78,7 +78,7 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| maxLength | int64 | | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 15 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** @@ -87,7 +87,7 @@ Description: Value that will be transformed const newValue = neosync.transformE164PhoneNumber(value, { preserveLength: false, - maxLength: 1, + maxLength: 15, seed: 1, }); @@ -773,7 +773,7 @@ Randomly generates one of the following genders: female (f), male (m), undefined | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | abbreviate | bool | false | false | Shortens length of generated value to 1. -| maxLength | int64 | 10000 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -783,7 +783,7 @@ Randomly generates one of the following genders: female (f), male (m), undefined const newValue = neosync.generateGender({ abbreviate: false, - maxLength: 10000, + maxLength: 100, seed: 1, }); diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index 08a1a455f0..2bdd40be61 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index f07b285bd5..819103523b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index 7f4929cc17..e48a35a859 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index 019c5e8f36..975fcf0f17 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index e9e57f0a85..df69dc011b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index b345233eb1..396a864db6 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index 93f93a123d..21c4c924c1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index 8976ee6273..89b2ba6875 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index 966fb1cf11..1e7bf1ce4b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index 20b290456b..20a9693908 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index ba2ca85e48..c7d63008ac 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) @@ -35,7 +34,7 @@ func NewGenerateGenderOpts( abbreviate = *abbreviateArg } - maxLength := int64(10000) + maxLength := int64(100) if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { maxLength = *maxLengthArg } @@ -71,7 +70,7 @@ func (t *GenerateGender) ParseOptions(opts map[string]any) (any, error) { maxLength, ok := opts["maxLength"].(int64) if !ok { - maxLength = 10000 + maxLength = 100 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index ef66fdf15a..07736c9d22 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index 342e058004..c2fb581bfb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index 7538e03f4d..693b6eee35 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index e29da788ce..a727585cd3 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index 6c3ea79150..9971041761 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index 29826543d1..9a32ff4464 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index 8d91bb6365..a171d165b7 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index c2673f117d..3d5acc9dd6 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index 9508e1f5de..f263bf0c08 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index dac7de62fb..e9c592b087 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index b8879ef7a8..2893408d73 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index a0e7d518b9..d2383feed4 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_uuid.go b/worker/pkg/benthos/transformers/gen_generate_uuid.go index 3b35d2bd6f..741c847386 100644 --- a/worker/pkg/benthos/transformers/gen_generate_uuid.go +++ b/worker/pkg/benthos/transformers/gen_generate_uuid.go @@ -4,8 +4,7 @@ package transformers -import ( - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" +import (transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) type GenerateUUID struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index cc324cf342..62f0d8ff96 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 66e1547a2e..91d646fe98 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index 7f6bcc4f58..280907faff 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) @@ -18,7 +17,7 @@ type TransformE164PhoneNumberOpts struct { randomizer rng.Rand preserveLength bool - maxLength *int64 + maxLength int64 } func NewTransformE164PhoneNumber() *TransformE164PhoneNumber { @@ -27,7 +26,7 @@ func NewTransformE164PhoneNumber() *TransformE164PhoneNumber { func NewTransformE164PhoneNumberOpts( preserveLengthArg *bool, - maxLength *int64, + maxLengthArg *int64, seedArg *int64, ) (*TransformE164PhoneNumberOpts, error) { preserveLength := bool(false) @@ -35,6 +34,11 @@ func NewTransformE164PhoneNumberOpts( preserveLength = *preserveLengthArg } + maxLength := int64(15) + if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + maxLength = *maxLengthArg + } + seed, err := transformer_utils.GetSeedOrDefault(seedArg) if err != nil { return nil, fmt.Errorf("unable to generate seed: %w", err) @@ -64,9 +68,9 @@ func (t *TransformE164PhoneNumber) ParseOptions(opts map[string]any) (any, error } transformerOpts.preserveLength = preserveLength - var maxLength *int64 - if arg, ok := opts["maxLength"].(int64); ok { - maxLength = &arg + maxLength, ok := opts["maxLength"].(int64) + if !ok { + maxLength = 15 } transformerOpts.maxLength = maxLength diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index f0c04720b5..22d92045e8 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index 4e613ccea3..9aee9f9bef 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index a29d4383c2..0d98ab2329 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index e6a7a50deb..b05327b280 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index ae1e15845a..c48b1068d0 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index 923fcc3168..4fa62bd6df 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index d78d634863..9650edc1ea 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index 49d1567b7d..9ad7bddb03 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index fbcd49d269..726ff6a619 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/nucleuscloud/neosync/worker/pkg/rng" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) diff --git a/worker/pkg/benthos/transformers/generate_gender.go b/worker/pkg/benthos/transformers/generate_gender.go index f8b89e9a28..e5313f56b4 100644 --- a/worker/pkg/benthos/transformers/generate_gender.go +++ b/worker/pkg/benthos/transformers/generate_gender.go @@ -14,7 +14,7 @@ func init() { spec := bloblang.NewPluginSpec(). Description("Randomly generates one of the following genders: female (f), male (m), undefined (u), nonbinary (n)."). Param(bloblang.NewBoolParam("abbreviate").Default(false).Description("Shortens length of generated value to 1.")). - Param(bloblang.NewInt64Param("max_length").Default(10000).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(100).Description("Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("generate_gender", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index b59bd2bfe9..3530324278 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -102,7 +102,9 @@ import ( {{- if eq .HasSeedParam true}} "github.com/nucleuscloud/neosync/worker/pkg/rng" {{ end }} + {{- if gt (len .FunctInfo.Params) 0 -}} transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + {{- end }} ) type {{.StructName}} struct{} diff --git a/worker/pkg/benthos/transformers/transform_e164_phone_number.go b/worker/pkg/benthos/transformers/transform_e164_phone_number.go index 5c48514779..8be8ece860 100644 --- a/worker/pkg/benthos/transformers/transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_e164_phone_number.go @@ -17,7 +17,7 @@ func init() { Description("Transforms an existing E164 formatted phone number."). Param(bloblang.NewAnyParam("value").Optional()). Param(bloblang.NewBoolParam("preserve_length").Default(false).Description("Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data.")). - Param(bloblang.NewInt64Param("max_length").Optional().Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). + Param(bloblang.NewInt64Param("max_length").Default(15).Description("Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters.")). Param(bloblang.NewInt64Param("seed").Optional().Description("An optional seed value used to generate deterministic outputs.")) err := bloblang.RegisterFunctionV2("transform_e164_phone_number", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { @@ -78,7 +78,7 @@ func (t *TransformE164PhoneNumber) Transform(value, opts any) (any, error) { return nil, errors.New("value is not a string") } - return transformE164PhoneNumber(parsedOpts.randomizer, valueStr, parsedOpts.preserveLength, parsedOpts.maxLength) + return transformE164PhoneNumber(parsedOpts.randomizer, valueStr, parsedOpts.preserveLength, &parsedOpts.maxLength) } // Generates a random phone number and returns it as a string diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 0abeb80d22..4f46c13e43 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -56,13 +56,10 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig: config := transformerConfig.GetGenerateCategoricalConfig() - fmt.Println(config.Categories) - opts, err := NewGenerateCategoricalOpts(&config.Categories, nil) if err != nil { return nil, err } - fmt.Println(opts.categories) generate := NewGenerateCategorical().Generate return &TransformerExecutor{ Opts: opts, @@ -130,7 +127,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform config := transformerConfig.GetGenerateEmailConfig() var emailType *string if config.EmailType != nil { - emailTypeStr := config.GetEmailType().String() + emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() emailType = &emailTypeStr } opts, err := NewGenerateEmailOpts(&maxLength, emailType, nil) @@ -149,12 +146,12 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform config := transformerConfig.GetTransformEmailConfig() var emailType *string if config.EmailType != nil { - emailTypeStr := config.GetEmailType().String() + emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() emailType = &emailTypeStr } var invalidEmailAction *string if config.InvalidEmailAction != nil { - invalidEmailActionStr := config.GetInvalidEmailAction().String() + invalidEmailActionStr := dtoInvalidEmailActionToTransformerInvalidEmailAction(config.GetInvalidEmailAction()).String() invalidEmailAction = &invalidEmailActionStr } var excludedDomains any = config.GetExcludedDomains() @@ -233,11 +230,15 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateFloat64Config: config := transformerConfig.GetGenerateFloat64Config() + var precision *int64 + if config.GetPrecision() != 0 { + precision = &config.Precision + } opts, err := NewGenerateFloat64Opts( &config.RandomizeSign, &config.Min, &config.Max, - &config.Precision, + precision, nil, // TODO: update scale based on colInfo if available nil, ) @@ -279,8 +280,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateGenderConfig: - ab := transformerConfig.GetGenerateGenderConfig().GetAbbreviate() - opts, err := NewGenerateGenderOpts(&ab, &maxLength, nil) + config := transformerConfig.GetGenerateGenderConfig() + opts, err := NewGenerateGenderOpts(&config.Abbreviate, &maxLength, nil) if err != nil { return nil, err } @@ -481,7 +482,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig: config := transformerConfig.GetTransformE164PhoneNumberConfig() - opts, err := NewTransformE164PhoneNumberOpts(&config.PreserveLength, &maxLength, nil) + opts, err := NewTransformE164PhoneNumberOpts(&config.PreserveLength, nil, nil) if err != nil { return nil, err } @@ -638,3 +639,25 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform return nil, fmt.Errorf("unsupported transformer: %v", transformerConfig) } } + +func dtoEmailTypeToTransformerEmailType(dto mgmtv1alpha1.GenerateEmailType) GenerateEmailType { + switch dto { + case mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME: + return GenerateEmailType_FullName + default: + return GenerateEmailType_UuidV4 + } +} + +func dtoInvalidEmailActionToTransformerInvalidEmailAction(dto mgmtv1alpha1.InvalidEmailAction) InvalidEmailAction { + switch dto { + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE: + return InvalidEmailAction_Generate + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_NULL: + return InvalidEmailAction_Null + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_PASSTHROUGH: + return InvalidEmailAction_Passthrough + default: + return InvalidEmailAction_Reject + } +} diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformers/transformer_initializer_test.go index e8460027a5..d10d5e27d0 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer_test.go +++ b/worker/pkg/benthos/transformers/transformer_initializer_test.go @@ -1,9 +1,13 @@ package transformers import ( + "strconv" "testing" + "time" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + presidioapi "github.com/nucleuscloud/neosync/internal/ee/presidio" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -63,6 +67,22 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformStringConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ + TransformStringConfig: &mgmtv1alpha1.TransformString{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test", executor.Opts) + require.NoError(t, err) + require.Len(t, *result.(*string), 4) + }) + + t.Run("TransformStringConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ TransformStringConfig: &mgmtv1alpha1.TransformString{}, @@ -73,17 +93,16 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotNil(t, executor) result, err := executor.Mutate("test", executor.Opts) require.NoError(t, err) - require.IsType(t, "test", result) - require.Len(t, result.(string), 4) + require.NotEmpty(t, result) }) t.Run("TransformInt64Config", func(t *testing.T) { - min, max := int64(1), int64(100) + rmin, rmax := int64(1), int64(5) config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ TransformInt64Config: &mgmtv1alpha1.TransformInt64{ - RandomizationRangeMin: min, - RandomizationRangeMax: max, + RandomizationRangeMin: rmin, + RandomizationRangeMax: rmax, }, }, } @@ -92,12 +111,40 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotNil(t, executor) result, err := executor.Mutate(int64(50), executor.Opts) require.NoError(t, err) - require.IsType(t, int64(0), result) - require.GreaterOrEqual(t, result.(int64), min) - require.LessOrEqual(t, result.(int64), max) + require.GreaterOrEqual(t, *result.(*int64), int64(48)) + require.LessOrEqual(t, *result.(*int64), int64(55)) + }) + + t.Run("TransformInt64Config_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ + TransformInt64Config: &mgmtv1alpha1.TransformInt64{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(int64(50), executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) }) t.Run("TransformFullNameConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ + TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("John Doe", executor.Opts) + require.NoError(t, err) + require.Len(t, *result.(*string), 8) + }) + t.Run("TransformFullNameConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{}, @@ -108,23 +155,63 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotNil(t, executor) result, err := executor.Mutate("John Doe", executor.Opts) require.NoError(t, err) - require.IsType(t, "John Doe", result) - require.Len(t, result.(string), 8) + require.NotEmpty(t, result) }) t.Run("GenerateEmailConfig", func(t *testing.T) { + emailType := mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateEmailConfig{ + GenerateEmailConfig: &mgmtv1alpha1.GenerateEmail{ + EmailType: &emailType, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + + t.Run("GenerateEmailConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ - Config: &mgmtv1alpha1.TransformerConfig_GenerateEmailConfig{}, + Config: &mgmtv1alpha1.TransformerConfig_GenerateEmailConfig{ + GenerateEmailConfig: &mgmtv1alpha1.GenerateEmail{}, + }, } executor, err := InitializeTransformerByConfigType(config) require.NoError(t, err) require.NotNil(t, executor) result, err := executor.Mutate(nil, executor.Opts) require.NoError(t, err) - require.Regexp(t, `^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$`, result) + require.NotEmpty(t, result) }) t.Run("TransformEmailConfig", func(t *testing.T) { + emailType := mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME + invalidEmailAction := mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ + TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ + PreserveDomain: true, + PreserveLength: true, + EmailType: &emailType, + InvalidEmailAction: &invalidEmailAction, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test@example.com", executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^[a-zA-Z0-9._%+-]+@example\.com$`, *result.(*string)) + require.Len(t, *result.(*string), 16) + }) + + t.Run("TransformEmailConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{}, @@ -135,11 +222,858 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotNil(t, executor) result, err := executor.Mutate("test@example.com", executor.Opts) require.NoError(t, err) - require.Regexp(t, `^[a-zA-Z0-9._%+-]+@example\.com$`, result) - require.Len(t, result.(string), 16) + require.NotEmpty(t, result) + }) + + t.Run("GenerateCardNumberConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ + GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{ + ValidLuhn: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^\d{13,19}$`, result) + }) + + t.Run("GenerateCardNumber_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ + GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^\d{13,19}$`, result) + }) + + t.Run("GenerateCityConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCityConfig{ + GenerateCityConfig: &mgmtv1alpha1.GenerateCity{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateE164PhoneNumberConfig", func(t *testing.T) { + rmin, rmax := int64(10), int64(10) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ + GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{ + Min: rmin, + Max: rmax, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^\+\d{10}$`, result) + }) + + t.Run("GenerateE164PhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ + GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + + t.Run("GenerateFirstNameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig{ + GenerateFirstNameConfig: &mgmtv1alpha1.GenerateFirstName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateFloat64Config", func(t *testing.T) { + randomizeSign := true + rmin, rmax := float64(-10), float64(10) + precision := int64(2) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{ + GenerateFloat64Config: &mgmtv1alpha1.GenerateFloat64{ + RandomizeSign: randomizeSign, + Min: rmin, + Max: rmax, + Precision: precision, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, float64(0), result) + floatResult := result.(float64) + require.GreaterOrEqual(t, floatResult, rmin) + require.LessOrEqual(t, floatResult, rmax) + }) + + t.Run("GenerateFloat64Config_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{ + GenerateFloat64Config: &mgmtv1alpha1.GenerateFloat64{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, float64(0), result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateFullAddressConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig{ + GenerateFullAddressConfig: &mgmtv1alpha1.GenerateFullAddress{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateFullNameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFullNameConfig{ + GenerateFullNameConfig: &mgmtv1alpha1.GenerateFullName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateGenderConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{ + GenerateGenderConfig: &mgmtv1alpha1.GenerateGender{ + Abbreviate: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Len(t, result, 1) + }) + + t.Run("GenerateGenderConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{ + GenerateGenderConfig: &mgmtv1alpha1.GenerateGender{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateInt64PhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64PhoneNumberConfig{ + GenerateInt64PhoneNumberConfig: &mgmtv1alpha1.GenerateInt64PhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.Regexp(t, `^[1-9]\d{9}$`, strconv.FormatInt(result.(int64), 10)) + }) + + t.Run("GenerateInt64Config", func(t *testing.T) { + randomizeSign := true + rmin, rmax := int64(-1000), int64(1000) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ + GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ + RandomizeSign: randomizeSign, + Min: rmin, + Max: rmax, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + intResult := result.(int64) + require.GreaterOrEqual(t, intResult, rmin) + require.LessOrEqual(t, intResult, rmax) + }) + + t.Run("GenerateInt64Config_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ + GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateLastNameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateLastNameConfig{ + GenerateLastNameConfig: &mgmtv1alpha1.GenerateLastName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Regexp(t, `^[A-Z][a-z]+$`, result) + }) + + t.Run("GenerateSha256HashConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateSha256HashConfig{ + GenerateSha256HashConfig: &mgmtv1alpha1.GenerateSha256Hash{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.Regexp(t, `^[a-f0-9]{64}$`, result) }) - // Add more subtests for other config types... + t.Run("GenerateSsnConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateSsnConfig{ + GenerateSsnConfig: &mgmtv1alpha1.GenerateSSN{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.Regexp(t, `^\d{3}-\d{2}-\d{4}$`, result) + }) + + t.Run("GenerateStateConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{ + GenerateStateConfig: &mgmtv1alpha1.GenerateState{ + GenerateFullName: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Regexp(t, `^[A-Z][a-z]+( [A-Z][a-z]+)*$`, result) + }) + + t.Run("GenerateStateConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{ + GenerateStateConfig: &mgmtv1alpha1.GenerateState{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Len(t, result, 2) + }) + + t.Run("GenerateStreetAddressConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig{ + GenerateStreetAddressConfig: &mgmtv1alpha1.GenerateStreetAddress{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateStringPhoneNumberConfig", func(t *testing.T) { + rmin, rmax := int64(10), int64(12) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{ + GenerateStringPhoneNumberConfig: &mgmtv1alpha1.GenerateStringPhoneNumber{ + Min: rmin, + Max: rmax, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.Regexp(t, `^\+?1?\d{10,12}$`, result) + }) + + t.Run("GenerateStringPhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{ + GenerateStringPhoneNumberConfig: &mgmtv1alpha1.GenerateStringPhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateStringConfig", func(t *testing.T) { + rmin, rmax := int64(5), int64(10) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ + GenerateStringConfig: &mgmtv1alpha1.GenerateString{ + Min: rmin, + Max: rmax, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.GreaterOrEqual(t, len(result.(string)), int(rmin)) + require.LessOrEqual(t, len(result.(string)), int(rmax)) + }) + + t.Run("GenerateStringConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ + GenerateStringConfig: &mgmtv1alpha1.GenerateString{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateUnixtimestampConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUnixtimestampConfig{ + GenerateUnixtimestampConfig: &mgmtv1alpha1.GenerateUnixTimestamp{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateUsernameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUsernameConfig{ + GenerateUsernameConfig: &mgmtv1alpha1.GenerateUsername{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateUtctimestampConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUtctimestampConfig{ + GenerateUtctimestampConfig: &mgmtv1alpha1.GenerateUtcTimestamp{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, time.Now(), result) + }) + + t.Run("GenerateUuidConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ + GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ + IncludeHyphens: false, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateUuidConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ + GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateZipcodeConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateZipcodeConfig{ + GenerateZipcodeConfig: &mgmtv1alpha1.GenerateZipcode{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + + t.Run("TransformE164PhoneNumberConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{ + TransformE164PhoneNumberConfig: &mgmtv1alpha1.TransformE164PhoneNumber{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "+12345678901" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalNumber, result) + require.Equal(t, len(originalNumber), len(*result.(*string))) + }) + + t.Run("TransformE164PhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{ + TransformE164PhoneNumberConfig: &mgmtv1alpha1.TransformE164PhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "+12345678901" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalNumber, result) + }) + + t.Run("TransformFirstNameConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ + TransformFirstNameConfig: &mgmtv1alpha1.TransformFirstName{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "John" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalName, result) + require.Equal(t, len(originalName), len(*result.(*string))) + }) + + t.Run("TransformFirstNameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ + TransformFirstNameConfig: &mgmtv1alpha1.TransformFirstName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "John" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalName, result) + require.NotEmpty(t, result) + }) + + t.Run("TransformFloat64Config", func(t *testing.T) { + randomizationRangeMin := float64(-10.0) + randomizationRangeMax := float64(10.0) + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ + TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{ + RandomizationRangeMin: randomizationRangeMin, + RandomizationRangeMax: randomizationRangeMax, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalValue := float64(5.5) + result, err := executor.Mutate(originalValue, executor.Opts) + require.NoError(t, err) + transformedValue := *result.(*float64) + require.NotEqual(t, originalValue, transformedValue) + require.GreaterOrEqual(t, transformedValue, originalValue+randomizationRangeMin) + require.LessOrEqual(t, transformedValue, originalValue+randomizationRangeMax) + }) + + t.Run("TransformFloat64Config_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ + TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalValue := float64(5.5) + result, err := executor.Mutate(originalValue, executor.Opts) + require.NoError(t, err) + transformedValue := *result.(*float64) + require.NotEqual(t, originalValue, transformedValue) + }) + + t.Run("TransformInt64PhoneNumberConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{ + TransformInt64PhoneNumberConfig: &mgmtv1alpha1.TransformInt64PhoneNumber{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := int64(1234567890) + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + transformedNumber := *result.(*int64) + require.NotEqual(t, originalNumber, transformedNumber) + require.GreaterOrEqual(t, transformedNumber, int64(1000000000)) + require.Less(t, transformedNumber, int64(10000000000)) + }) + + t.Run("TransformInt64PhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{ + TransformInt64PhoneNumberConfig: &mgmtv1alpha1.TransformInt64PhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := int64(1234567890) + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + transformedNumber := *result.(*int64) + require.NotEqual(t, originalNumber, transformedNumber) + require.NotEmpty(t, transformedNumber) + }) + + t.Run("TransformLastNameConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{ + TransformLastNameConfig: &mgmtv1alpha1.TransformLastName{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "Smith" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalName, result) + require.Equal(t, len(originalName), len(*result.(*string))) + }) + + t.Run("TransformLastNameConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{ + TransformLastNameConfig: &mgmtv1alpha1.TransformLastName{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "Smith" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalName, result) + }) + + t.Run("TransformPhoneNumberConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{ + TransformPhoneNumberConfig: &mgmtv1alpha1.TransformPhoneNumber{ + PreserveLength: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "123-456-7890" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalNumber, result) + require.Equal(t, len(originalNumber), len(*result.(*string))) + }) + + t.Run("TransformPhoneNumberConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{ + TransformPhoneNumberConfig: &mgmtv1alpha1.TransformPhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "123-456-7890" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalNumber, result) + }) + + t.Run("Nullconfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_Nullconfig{ + Nullconfig: &mgmtv1alpha1.Null{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("any value", executor.Opts) + require.NoError(t, err) + require.Equal(t, "null", result) + }) + + t.Run("GenerateDefaultConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateDefaultConfig{ + GenerateDefaultConfig: &mgmtv1alpha1.GenerateDefault{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("any value", executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + + t.Run("TransformCharacterScrambleConfig", func(t *testing.T) { + userProvidedRegex := `[a-zA-Z]` + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformCharacterScrambleConfig{ + TransformCharacterScrambleConfig: &mgmtv1alpha1.TransformCharacterScramble{ + UserProvidedRegex: &userProvidedRegex, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalString := "Hello123World" + result, err := executor.Mutate(originalString, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalString, result) + require.Equal(t, len(originalString), len(*result.(*string))) + }) + + t.Run("TransformCharacterScrambleConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformCharacterScrambleConfig{ + TransformCharacterScrambleConfig: &mgmtv1alpha1.TransformCharacterScramble{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalString := "Hello123World" + result, err := executor.Mutate(originalString, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalString, result) + require.NotEmpty(t, result) + }) + + t.Run("GenerateCountryConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{ + GenerateCountryConfig: &mgmtv1alpha1.GenerateCountry{ + GenerateFullName: true, + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Greater(t, len(result.(string)), 2) + }) + + t.Run("GenerateCountryConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{ + GenerateCountryConfig: &mgmtv1alpha1.GenerateCountry{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.LessOrEqual(t, len(result.(string)), 2) + }) + + t.Run("TransformPiiTextConfig_Empty", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformPiiTextConfig{ + TransformPiiTextConfig: &mgmtv1alpha1.TransformPiiText{}, + }, + } + + mockanalyze := presidioapi.NewMockAnalyzeInterface(t) + mockanon := presidioapi.NewMockAnonymizeInterface(t) + + mockanalyze.On("PostAnalyzeWithResponse", mock.Anything, mock.Anything). + Return(&presidioapi.PostAnalyzeResponse{ + JSON200: &[]presidioapi.RecognizerResultWithAnaysisExplanation{ + {}, + }, + }, nil) + + mockText := "bar" + mockanon.On("PostAnonymizeWithResponse", mock.Anything, mock.Anything). + Return(&presidioapi.PostAnonymizeResponse{ + JSON200: &presidioapi.AnonymizeResponse{Text: &mockText}, + }, nil) + + execOpts := []TransformerExecutorOption{ + WithTransformPiiTextConfig(mockanalyze, mockanon), + } + executor, err := InitializeTransformerByConfigType(config, execOpts...) + require.NoError(t, err) + require.NotNil(t, executor) + + originalText := "Hello, John Doe!" + result, err := executor.Mutate(originalText, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEqual(t, originalText, result) + require.Equal(t, mockText, result) + }) t.Run("UnsupportedConfig", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ From 3da7202021827231c0edabe3d6cd3b2158279e36 Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 12:30:40 -0700 Subject: [PATCH 04/32] change generate_random_string to match proto --- .../gen-javascript-transformer.md | 62 +++++++++---------- .../apps/web/@types/neosync-transformers.d.ts | 30 ++++----- ...andom_string.go => gen_generate_string.go} | 24 +++---- .../transformers/gen_neosync_transformers.go | 2 +- ...te_random_string.go => generate_string.go} | 6 +- ...string_test.go => generate_string_test.go} | 0 .../transformers/transformer_initializer.go | 4 +- 7 files changed, 64 insertions(+), 64 deletions(-) rename worker/pkg/benthos/transformers/{gen_generate_random_string.go => gen_generate_string.go} (71%) rename worker/pkg/benthos/transformers/{generate_random_string.go => generate_string.go} (90%) rename worker/pkg/benthos/transformers/{generate_random_string_test.go => generate_string_test.go} (100%) diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index fb6aea2703..cd50b55574 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -920,12 +920,12 @@ const newValue = neosync.generateLastName({ -### generateRandomString +### generateSHA256Hash -Generates a random string of alphanumeric characters.. +Generates a random SHA256 hash and returns it as a string. **Parameters** @@ -933,32 +933,25 @@ Generates a random string of alphanumeric characters.. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| min | int64 | 1 | false | Specifies the minimum length for the generated string. -| max | int64 | 100 | false | Specifies the maximum length for the generated string. -| seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.generateRandomString({ - min: 1, - max: 100, - seed: 1, -}); +const newValue = neosync.generateSHA256Hash({}); ```
-### generateSHA256Hash +### generateSSN -Generates a random SHA256 hash and returns it as a string. +Generates a random social security numbers including the hyphens in the format xxx-xx-xxxx. **Parameters** @@ -966,25 +959,28 @@ Generates a random SHA256 hash and returns it as a string. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | +| seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.generateSHA256Hash({}); +const newValue = neosync.generateSSN({ + seed: 1, +}); ```
-### generateSSN +### generateState -Generates a random social security numbers including the hyphens in the format xxx-xx-xxxx. +Randomly selects a US state and by default, returns it as a 2-letter state code. **Parameters** @@ -992,6 +988,7 @@ Generates a random social security numbers including the hyphens in the format x | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | +| generateFullName | bool | false | false | If true returns the full state name instead of the two character state code. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -999,7 +996,8 @@ Generates a random social security numbers including the hyphens in the format x ```javascript -const newValue = neosync.generateSSN({ +const newValue = neosync.generateState({ + generateFullName: false, seed: 1, }); @@ -1008,12 +1006,12 @@ const newValue = neosync.generateSSN({ -### generateState +### generateStreetAddress -Randomly selects a US state and by default, returns it as a 2-letter state code. +Randomly generates a street address. **Parameters** @@ -1021,7 +1019,7 @@ Randomly selects a US state and by default, returns it as a 2-letter state code. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| generateFullName | bool | false | false | If true returns the full state name instead of the two character state code. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1029,8 +1027,8 @@ Randomly selects a US state and by default, returns it as a 2-letter state code. ```javascript -const newValue = neosync.generateState({ - generateFullName: false, +const newValue = neosync.generateStreetAddress({ + maxLength: 100, seed: 1, }); @@ -1039,12 +1037,12 @@ const newValue = neosync.generateState({ -### generateStreetAddress +### generateString -Randomly generates a street address. +Generates a random string of alphanumeric characters.. **Parameters** @@ -1052,7 +1050,8 @@ Randomly generates a street address. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| min | int64 | 1 | false | Specifies the minimum length for the generated string. +| max | int64 | 100 | false | Specifies the maximum length for the generated string. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1060,8 +1059,9 @@ Randomly generates a street address. ```javascript -const newValue = neosync.generateStreetAddress({ - maxLength: 100, +const newValue = neosync.generateString({ + min: 1, + max: 100, seed: 1, }); diff --git a/frontend/apps/web/@types/neosync-transformers.d.ts b/frontend/apps/web/@types/neosync-transformers.d.ts index 3c2c4ea0a4..3da1a748b7 100644 --- a/frontend/apps/web/@types/neosync-transformers.d.ts +++ b/frontend/apps/web/@types/neosync-transformers.d.ts @@ -402,21 +402,6 @@ declare namespace neosync { declare function generateLastName(options: GenerateLastNameOptions): any; - export interface GenerateRandomStringOptions { - /** Specifies the minimum length for the generated string. */ - min?: number; - /** Specifies the maximum length for the generated string. */ - max?: number; - /** An optional seed value used to generate deterministic outputs. */ - seed?: number; - } - - /** - * Generates a random string of alphanumeric characters.. - */ - declare function generateRandomString(options: GenerateRandomStringOptions): any; - - export interface GenerateSHA256HashOptions { } @@ -463,6 +448,21 @@ declare namespace neosync { declare function generateStreetAddress(options: GenerateStreetAddressOptions): any; + export interface GenerateStringOptions { + /** Specifies the minimum length for the generated string. */ + min?: number; + /** Specifies the maximum length for the generated string. */ + max?: number; + /** An optional seed value used to generate deterministic outputs. */ + seed?: number; + } + + /** + * Generates a random string of alphanumeric characters.. + */ + declare function generateString(options: GenerateStringOptions): any; + + export interface GenerateStringPhoneNumberOptions { /** Specifies the minimum length for the generated phone number. */ min?: number; diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_string.go similarity index 71% rename from worker/pkg/benthos/transformers/gen_generate_random_string.go rename to worker/pkg/benthos/transformers/gen_generate_string.go index 9971041761..506c2b0566 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_string.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: generate_random_string.go +// source: generate_string.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type GenerateRandomString struct{} +type GenerateString struct{} -type GenerateRandomStringOpts struct { +type GenerateStringOpts struct { randomizer rng.Rand min int64 max int64 } -func NewGenerateRandomString() *GenerateRandomString { - return &GenerateRandomString{} +func NewGenerateString() *GenerateString { + return &GenerateString{} } -func NewGenerateRandomStringOpts( +func NewGenerateStringOpts( minArg *int64, maxArg *int64, seedArg *int64, -) (*GenerateRandomStringOpts, error) { +) (*GenerateStringOpts, error) { min := int64(1) if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { min = *minArg @@ -44,23 +44,23 @@ func NewGenerateRandomStringOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &GenerateRandomStringOpts{ + return &GenerateStringOpts{ min: min, max: max, randomizer: rng.New(seed), }, nil } -func (t *GenerateRandomString) GetJsTemplateData() (*TemplateData, error) { +func (t *GenerateString) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "generateRandomString", + Name: "generateString", Description: "Generates a random string of alphanumeric characters..", Example: "", }, nil } -func (t *GenerateRandomString) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &GenerateRandomStringOpts{} +func (t *GenerateString) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &GenerateStringOpts{} min, ok := opts["min"].(int64) if !ok { diff --git a/worker/pkg/benthos/transformers/gen_neosync_transformers.go b/worker/pkg/benthos/transformers/gen_neosync_transformers.go index 6a256fe070..8cb731bfb9 100644 --- a/worker/pkg/benthos/transformers/gen_neosync_transformers.go +++ b/worker/pkg/benthos/transformers/gen_neosync_transformers.go @@ -36,11 +36,11 @@ func GetNeosyncGenerators() []NeosyncGenerator { NewGenerateInt64PhoneNumber(), NewGenerateInternationalPhoneNumber(), NewGenerateLastName(), - NewGenerateRandomString(), NewGenerateSHA256Hash(), NewGenerateSSN(), NewGenerateState(), NewGenerateStreetAddress(), + NewGenerateString(), NewGenerateStringPhoneNumber(), NewGenerateUnixTimestamp(), NewGenerateUsername(), diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_string.go similarity index 90% rename from worker/pkg/benthos/transformers/generate_random_string.go rename to worker/pkg/benthos/transformers/generate_string.go index 69b1bdad36..a632490aa1 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_string.go @@ -8,7 +8,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:generate:generateRandomString +// +neosyncTransformerBuilder:generate:generateString func init() { spec := bloblang.NewPluginSpec(). @@ -54,8 +54,8 @@ func init() { } } -func (t *GenerateRandomString) Generate(opts any) (any, error) { - parsedOpts, ok := opts.(*GenerateRandomStringOpts) +func (t *GenerateString) Generate(opts any) (any, error) { + parsedOpts, ok := opts.(*GenerateStringOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/generate_random_string_test.go b/worker/pkg/benthos/transformers/generate_string_test.go similarity index 100% rename from worker/pkg/benthos/transformers/generate_random_string_test.go rename to worker/pkg/benthos/transformers/generate_string_test.go diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 4f46c13e43..df07eb7b7b 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -402,11 +402,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateStringOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } - generate := NewGenerateRandomString().Generate + generate := NewGenerateString().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { From 20d5ef0e8747c32b4bb4af7d961eb33a67bc8712 Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 12:40:27 -0700 Subject: [PATCH 05/32] change generate_e164_phone_number to match proto --- .../gen-javascript-transformer.md | 66 +++++++++---------- .../apps/web/@types/neosync-transformers.d.ts | 30 ++++----- ...r.go => gen_generate_e164_phone_number.go} | 24 +++---- .../transformers/gen_neosync_transformers.go | 2 +- ...umber.go => generate_e164_phone_number.go} | 6 +- ....go => generate_e164_phone_number_test.go} | 0 .../transformers/transformer_initializer.go | 4 +- 7 files changed, 66 insertions(+), 66 deletions(-) rename worker/pkg/benthos/transformers/{gen_generate_international_phone_number.go => gen_generate_e164_phone_number.go} (66%) rename worker/pkg/benthos/transformers/{generate_international_phone_number.go => generate_e164_phone_number.go} (91%) rename worker/pkg/benthos/transformers/{generate_international_phone_number_test.go => generate_e164_phone_number_test.go} (100%) diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index cd50b55574..ab0c62af03 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -593,6 +593,39 @@ const newValue = neosync.generateCountry({
+ + +### generateE164PhoneNumber + +Generates a new random international phone number including the + sign and no hyphens. + +**Parameters** + +**Config** + +| Field | Type | Default | Required | Description | +| -------- | ---- | ------- | -------- | ----------- | +| min | int64 | 9 | false | Specifies the minimum value for the generated phone number. +| max | int64 | 15 | false | Specifies the maximum value for the generated phone number. +| seed | int64 | | false | An optional seed value used to generate deterministic outputs. +
+ +**Example** + +```javascript + +const newValue = neosync.generateE164PhoneNumber({ + min: 9, + max: 15, + seed: 1, +}); + +``` +
+ + @@ -855,39 +888,6 @@ const newValue = neosync.generateInt64PhoneNumber({
- - -### generateInternationalPhoneNumber - -Generates a new random international phone number including the + sign and no hyphens. - -**Parameters** - -**Config** - -| Field | Type | Default | Required | Description | -| -------- | ---- | ------- | -------- | ----------- | -| min | int64 | 9 | false | Specifies the minimum value for the generated phone number. -| max | int64 | 15 | false | Specifies the maximum value for the generated phone number. -| seed | int64 | | false | An optional seed value used to generate deterministic outputs. -
- -**Example** - -```javascript - -const newValue = neosync.generateInternationalPhoneNumber({ - min: 9, - max: 15, - seed: 1, -}); - -``` -
- - diff --git a/frontend/apps/web/@types/neosync-transformers.d.ts b/frontend/apps/web/@types/neosync-transformers.d.ts index 3da1a748b7..21c06519d8 100644 --- a/frontend/apps/web/@types/neosync-transformers.d.ts +++ b/frontend/apps/web/@types/neosync-transformers.d.ts @@ -256,6 +256,21 @@ declare namespace neosync { declare function generateCountry(options: GenerateCountryOptions): any; + export interface GenerateE164PhoneNumberOptions { + /** Specifies the minimum value for the generated phone number. */ + min?: number; + /** Specifies the maximum value for the generated phone number. */ + max?: number; + /** An optional seed value used to generate deterministic outputs. */ + seed?: number; + } + + /** + * Generates a new random international phone number including the + sign and no hyphens. + */ + declare function generateE164PhoneNumber(options: GenerateE164PhoneNumberOptions): any; + + export interface GenerateEmailOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; @@ -374,21 +389,6 @@ declare namespace neosync { declare function generateInt64PhoneNumber(options: GenerateInt64PhoneNumberOptions): any; - export interface GenerateInternationalPhoneNumberOptions { - /** Specifies the minimum value for the generated phone number. */ - min?: number; - /** Specifies the maximum value for the generated phone number. */ - max?: number; - /** An optional seed value used to generate deterministic outputs. */ - seed?: number; - } - - /** - * Generates a new random international phone number including the + sign and no hyphens. - */ - declare function generateInternationalPhoneNumber(options: GenerateInternationalPhoneNumberOptions): any; - - export interface GenerateLastNameOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go similarity index 66% rename from worker/pkg/benthos/transformers/gen_generate_international_phone_number.go rename to worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go index 693b6eee35..3fcdbd5575 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: generate_international_phone_number.go +// source: generate_e164_phone_number.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type GenerateInternationalPhoneNumber struct{} +type GenerateE164PhoneNumber struct{} -type GenerateInternationalPhoneNumberOpts struct { +type GenerateE164PhoneNumberOpts struct { randomizer rng.Rand min int64 max int64 } -func NewGenerateInternationalPhoneNumber() *GenerateInternationalPhoneNumber { - return &GenerateInternationalPhoneNumber{} +func NewGenerateE164PhoneNumber() *GenerateE164PhoneNumber { + return &GenerateE164PhoneNumber{} } -func NewGenerateInternationalPhoneNumberOpts( +func NewGenerateE164PhoneNumberOpts( minArg *int64, maxArg *int64, seedArg *int64, -) (*GenerateInternationalPhoneNumberOpts, error) { +) (*GenerateE164PhoneNumberOpts, error) { min := int64(9) if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { min = *minArg @@ -44,23 +44,23 @@ func NewGenerateInternationalPhoneNumberOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &GenerateInternationalPhoneNumberOpts{ + return &GenerateE164PhoneNumberOpts{ min: min, max: max, randomizer: rng.New(seed), }, nil } -func (t *GenerateInternationalPhoneNumber) GetJsTemplateData() (*TemplateData, error) { +func (t *GenerateE164PhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "generateInternationalPhoneNumber", + Name: "generateE164PhoneNumber", Description: "Generates a new random international phone number including the + sign and no hyphens.", Example: "", }, nil } -func (t *GenerateInternationalPhoneNumber) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &GenerateInternationalPhoneNumberOpts{} +func (t *GenerateE164PhoneNumber) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &GenerateE164PhoneNumberOpts{} min, ok := opts["min"].(int64) if !ok { diff --git a/worker/pkg/benthos/transformers/gen_neosync_transformers.go b/worker/pkg/benthos/transformers/gen_neosync_transformers.go index 8cb731bfb9..6bc9d18b7d 100644 --- a/worker/pkg/benthos/transformers/gen_neosync_transformers.go +++ b/worker/pkg/benthos/transformers/gen_neosync_transformers.go @@ -26,6 +26,7 @@ func GetNeosyncGenerators() []NeosyncGenerator { NewGenerateCategorical(), NewGenerateCity(), NewGenerateCountry(), + NewGenerateE164PhoneNumber(), NewGenerateEmail(), NewGenerateFirstName(), NewGenerateFloat64(), @@ -34,7 +35,6 @@ func GetNeosyncGenerators() []NeosyncGenerator { NewGenerateGender(), NewGenerateInt64(), NewGenerateInt64PhoneNumber(), - NewGenerateInternationalPhoneNumber(), NewGenerateLastName(), NewGenerateSHA256Hash(), NewGenerateSSN(), diff --git a/worker/pkg/benthos/transformers/generate_international_phone_number.go b/worker/pkg/benthos/transformers/generate_e164_phone_number.go similarity index 91% rename from worker/pkg/benthos/transformers/generate_international_phone_number.go rename to worker/pkg/benthos/transformers/generate_e164_phone_number.go index 715f48cc17..0fd0ff4a48 100644 --- a/worker/pkg/benthos/transformers/generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_e164_phone_number.go @@ -10,7 +10,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:generate:generateInternationalPhoneNumber +// +neosyncTransformerBuilder:generate:generateE164PhoneNumber func init() { spec := bloblang.NewPluginSpec(). @@ -56,8 +56,8 @@ func init() { } } -func (t *GenerateInternationalPhoneNumber) Generate(opts any) (any, error) { - parsedOpts, ok := opts.(*GenerateInternationalPhoneNumberOpts) +func (t *GenerateE164PhoneNumber) Generate(opts any) (any, error) { + parsedOpts, ok := opts.(*GenerateE164PhoneNumberOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/generate_international_phone_number_test.go b/worker/pkg/benthos/transformers/generate_e164_phone_number_test.go similarity index 100% rename from worker/pkg/benthos/transformers/generate_international_phone_number_test.go rename to worker/pkg/benthos/transformers/generate_e164_phone_number_test.go diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index df07eb7b7b..238aa13720 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -204,11 +204,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: config := transformerConfig.GetGenerateE164PhoneNumberConfig() - opts, err := NewGenerateInternationalPhoneNumberOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateE164PhoneNumberOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } - generate := NewGenerateInternationalPhoneNumber().Generate + generate := NewGenerateE164PhoneNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { From 78d92f49d3cdebcd9341875ceff63f45e94728bd Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 12:50:15 -0700 Subject: [PATCH 06/32] change transform_string_phone_number to match proto --- .../gen-javascript-transformer.md | 24 +++++++++---------- .../apps/web/@types/neosync-transformers.d.ts | 20 ++++++++-------- .../transformers/gen_neosync_transformers.go | 2 +- ...umber.go => gen_transform_phone_number.go} | 24 +++++++++---------- ...ne_number.go => transform_phone_number.go} | 6 ++--- ...test.go => transform_phone_number_test.go} | 0 .../transformers/transformer_initializer.go | 4 ++-- 7 files changed, 40 insertions(+), 40 deletions(-) rename worker/pkg/benthos/transformers/{gen_transform_string_phone_number.go => gen_transform_phone_number.go} (72%) rename worker/pkg/benthos/transformers/{transform_string_phone_number.go => transform_phone_number.go} (93%) rename worker/pkg/benthos/transformers/{transform_string_phone_number_test.go => transform_phone_number_test.go} (100%) diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index ab0c62af03..dbe5e0346b 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -358,12 +358,12 @@ const newValue = neosync.transformLastName(value, { -### transformString +### transformPhoneNumber -Anonymizes and transforms an existing string value. +Anonymizes and transforms an existing phone number that is typed as a string. **Parameters** @@ -376,17 +376,15 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| minLength | int64 | 1 | false | Specifies the minimum length of the transformed value. -| maxLength | int64 | 100 | false | Specifies the maximum length of the transformed value. +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.transformString(value, { +const newValue = neosync.transformPhoneNumber(value, { preserveLength: false, - minLength: 1, maxLength: 100, seed: 1, }); @@ -396,12 +394,12 @@ const newValue = neosync.transformString(value, { -### transformStringPhoneNumber +### transformString -Anonymizes and transforms an existing phone number that is typed as a string. +Anonymizes and transforms an existing string value. **Parameters** @@ -414,15 +412,17 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| minLength | int64 | 1 | false | Specifies the minimum length of the transformed value. +| maxLength | int64 | 100 | false | Specifies the maximum length of the transformed value. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.transformStringPhoneNumber(value, { +const newValue = neosync.transformString(value, { preserveLength: false, + minLength: 1, maxLength: 100, seed: 1, }); diff --git a/frontend/apps/web/@types/neosync-transformers.d.ts b/frontend/apps/web/@types/neosync-transformers.d.ts index 21c06519d8..bde04a5b04 100644 --- a/frontend/apps/web/@types/neosync-transformers.d.ts +++ b/frontend/apps/web/@types/neosync-transformers.d.ts @@ -155,36 +155,36 @@ declare namespace neosync { declare function transformLastName(value: any, options: TransformLastNameOptions): any; - export interface TransformStringOptions { + export interface TransformPhoneNumberOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ preserveLength?: boolean; - /** Specifies the minimum length of the transformed value. */ - minLength?: number; - /** Specifies the maximum length of the transformed value. */ + /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } /** - * Anonymizes and transforms an existing string value. + * Anonymizes and transforms an existing phone number that is typed as a string. */ - declare function transformString(value: any, options: TransformStringOptions): any; + declare function transformPhoneNumber(value: any, options: TransformPhoneNumberOptions): any; - export interface TransformStringPhoneNumberOptions { + export interface TransformStringOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ preserveLength?: boolean; - /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ + /** Specifies the minimum length of the transformed value. */ + minLength?: number; + /** Specifies the maximum length of the transformed value. */ maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } /** - * Anonymizes and transforms an existing phone number that is typed as a string. + * Anonymizes and transforms an existing string value. */ - declare function transformStringPhoneNumber(value: any, options: TransformStringPhoneNumberOptions): any; + declare function transformString(value: any, options: TransformStringOptions): any; diff --git a/worker/pkg/benthos/transformers/gen_neosync_transformers.go b/worker/pkg/benthos/transformers/gen_neosync_transformers.go index 6bc9d18b7d..f9f5e67746 100644 --- a/worker/pkg/benthos/transformers/gen_neosync_transformers.go +++ b/worker/pkg/benthos/transformers/gen_neosync_transformers.go @@ -14,8 +14,8 @@ func GetNeosyncTransformers() []NeosyncTransformer { NewTransformInt64(), NewTransformInt64PhoneNumber(), NewTransformLastName(), + NewTransformPhoneNumber(), NewTransformString(), - NewTransformStringPhoneNumber(), } } diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_phone_number.go similarity index 72% rename from worker/pkg/benthos/transformers/gen_transform_string_phone_number.go rename to worker/pkg/benthos/transformers/gen_transform_phone_number.go index 726ff6a619..44f86d0b29 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_phone_number.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: transform_string_phone_number.go +// source: transform_phone_number.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type TransformStringPhoneNumber struct{} +type TransformPhoneNumber struct{} -type TransformStringPhoneNumberOpts struct { +type TransformPhoneNumberOpts struct { randomizer rng.Rand preserveLength bool maxLength int64 } -func NewTransformStringPhoneNumber() *TransformStringPhoneNumber { - return &TransformStringPhoneNumber{} +func NewTransformPhoneNumber() *TransformPhoneNumber { + return &TransformPhoneNumber{} } -func NewTransformStringPhoneNumberOpts( +func NewTransformPhoneNumberOpts( preserveLengthArg *bool, maxLengthArg *int64, seedArg *int64, -) (*TransformStringPhoneNumberOpts, error) { +) (*TransformPhoneNumberOpts, error) { preserveLength := bool(false) if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg @@ -44,23 +44,23 @@ func NewTransformStringPhoneNumberOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &TransformStringPhoneNumberOpts{ + return &TransformPhoneNumberOpts{ preserveLength: preserveLength, maxLength: maxLength, randomizer: rng.New(seed), }, nil } -func (t *TransformStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) { +func (t *TransformPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "transformStringPhoneNumber", + Name: "transformPhoneNumber", Description: "Anonymizes and transforms an existing phone number that is typed as a string.", Example: "", }, nil } -func (t *TransformStringPhoneNumber) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &TransformStringPhoneNumberOpts{} +func (t *TransformPhoneNumber) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &TransformPhoneNumberOpts{} preserveLength, ok := opts["preserveLength"].(bool) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_string_phone_number.go b/worker/pkg/benthos/transformers/transform_phone_number.go similarity index 93% rename from worker/pkg/benthos/transformers/transform_string_phone_number.go rename to worker/pkg/benthos/transformers/transform_phone_number.go index e587089f12..0e2a5fbe74 100644 --- a/worker/pkg/benthos/transformers/transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_phone_number.go @@ -9,7 +9,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:transform:transformStringPhoneNumber +// +neosyncTransformerBuilder:transform:transformPhoneNumber func init() { spec := bloblang.NewPluginSpec(). @@ -61,8 +61,8 @@ func init() { } } -func (t *TransformStringPhoneNumber) Transform(value, opts any) (any, error) { - parsedOpts, ok := opts.(*TransformStringPhoneNumberOpts) +func (t *TransformPhoneNumber) Transform(value, opts any) (any, error) { + parsedOpts, ok := opts.(*TransformPhoneNumberOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/transform_string_phone_number_test.go b/worker/pkg/benthos/transformers/transform_phone_number_test.go similarity index 100% rename from worker/pkg/benthos/transformers/transform_string_phone_number_test.go rename to worker/pkg/benthos/transformers/transform_phone_number_test.go diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 238aa13720..c4576a4e6f 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -558,11 +558,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformStringPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) + opts, err := NewTransformPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) if err != nil { return nil, err } - transform := NewTransformStringPhoneNumber().Transform + transform := NewTransformPhoneNumber().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { From 788dad2f9e1620caa041abd972b9157742b31f07 Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 12:53:32 -0700 Subject: [PATCH 07/32] fix lint --- worker/pkg/benthos/transformers/generate_email_test.go | 1 - worker/pkg/benthos/transformers/generate_state_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/worker/pkg/benthos/transformers/generate_email_test.go b/worker/pkg/benthos/transformers/generate_email_test.go index d1d55ec61f..7608f7ef4c 100644 --- a/worker/pkg/benthos/transformers/generate_email_test.go +++ b/worker/pkg/benthos/transformers/generate_email_test.go @@ -89,5 +89,4 @@ func Test_RandomEmailTransformer_NoOptions(t *testing.T) { require.NotEmpty(t, resStr) require.NotEmptyf(t, resStr, fmt.Sprintf("The email should be less than or equal to the max length. This is the error email:%s", res)) - } diff --git a/worker/pkg/benthos/transformers/generate_state_test.go b/worker/pkg/benthos/transformers/generate_state_test.go index bc9788c912..1fe4fb9ca5 100644 --- a/worker/pkg/benthos/transformers/generate_state_test.go +++ b/worker/pkg/benthos/transformers/generate_state_test.go @@ -90,5 +90,4 @@ func Test_StateTransformer_NoOptions(t *testing.T) { res, err := ex.Query(nil) assert.NoError(t, err) assert.NotEmpty(t, res) - } From a9a32c9e6a3506d551fe311f7af4e54998887032 Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 13:23:24 -0700 Subject: [PATCH 08/32] fix javascript integration test --- .../javascript-transformers/create.sql | 1 - .../javascript-transformers/insert.sql | 28 +++++++++---------- .../javascript-transformers/job_mappings.go | 8 ------ .../testdata/javascript-transformers/tests.go | 7 ++--- 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql index c78345c920..968a835601 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql @@ -20,7 +20,6 @@ CREATE TABLE transformers ( city VARCHAR(255), full_address VARCHAR(255), gender VARCHAR(255), - international_phone VARCHAR(255), sha256 VARCHAR(255), ssn VARCHAR(255), state VARCHAR(255), diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql index b823a23b99..a0a6b498e8 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql @@ -2,19 +2,19 @@ SET search_path TO javascript; INSERT INTO transformers ( e164_phone_number, email, measurement, int64, int64_phone_number, string_phone_number, first_name, last_name, full_name, str, character_scramble, bool, card_number, categorical, - city, full_address, gender, international_phone, sha256, ssn, state, street_address, + city, full_address, gender, sha256, ssn, state, street_address, unix_time, username, utc_timestamp, uuid, zipcode ) VALUES - ('+12345678901', 'user1@example.com', 1.1, 1000001, 12345678901, '123-456-7890', 'John', 'Doe', 'John Doe', 'Sample string 1', 'abcd1234', true, '1234567812345678', 'Category1', 'City1', '123 Main St, City1', 'Male', '+12345678901', 'sha256_1', '123-45-6789', 'State1', '123 Main St', 1617281823, 'user1', '2024-05-15T13:51:01.657574Z', 'uuid_1', 12345), - ('+12345678902', 'user2@example.com', 2.2, 1000002, 12345678902, '123-456-7891', 'Jane', 'Smith', 'Jane Smith', 'Sample string 2', 'abcd1235', false, '1234567812345679', 'Category2', 'City2', '456 Main St, City2', 'Female', '+12345678902', 'sha256_2', '123-45-6790', 'State2', '456 Main St', 1617281824, 'user2', '2024-05-15T13:51:01.657574Z', 'uuid_2', 12346), - ('+12345678903', 'user3@example.com', 3.3, 1000003, 12345678903, '123-456-7892', 'Alice', 'Johnson', 'Alice Johnson', 'Sample string 3', 'abcd1236', true, '1234567812345680', 'Category3', 'City3', '789 Main St, City3', 'Female', '+12345678903', 'sha256_3', '123-45-6791', 'State3', '789 Main St', 1617281825, 'user3', '2024-05-15T13:51:01.657574Z', 'uuid_3', 12347), - ('+12345678904', 'user4@example.com', 4.4, 1000004, 12345678904, '123-456-7893', 'Bob', 'Brown', 'Bob Brown', 'Sample string 4', 'abcd1237', false, '1234567812345681', 'Category4', 'City4', '101 Main St, City4', 'Male', '+12345678904', 'sha256_4', '123-45-6792', 'State4', '101 Main St', 1617281826, 'user4', '2024-05-15T13:51:01.657574Z', 'uuid_4', 12348), - ('+12345678905', 'user5@example.com', 5.5, 1000005, 12345678905, '123-456-7894', 'Charlie', 'Davis', 'Charlie Davis', 'Sample string 5', 'abcd1238', true, '1234567812345682', 'Category5', 'City5', '202 Main St, City5', 'Male', '+12345678905', 'sha256_5', '123-45-6793', 'State5', '202 Main St', 1617281827, 'user5', '2024-05-15T13:51:01.657574Z', 'uuid_5', 12349), - ('+12345678906', 'user6@example.com', 6.6, 1000006, 12345678906, '123-456-7895', 'David', 'Wilson', 'David Wilson', 'Sample string 6', 'abcd1239', false, '1234567812345683', 'Category6', 'City6', '303 Main St, City6', 'Male', '+12345678906', 'sha256_6', '123-45-6794', 'State6', '303 Main St', 1617281828, 'user6', '2024-05-15T13:51:01.657574Z', 'uuid_6', 12350), - ('+12345678907', 'user7@example.com', 7.7, 1000007, 12345678907, '123-456-7896', 'Eve', 'Martinez', 'Eve Martinez', 'Sample string 7', 'abcd1240', true, '1234567812345684', 'Category7', 'City7', '404 Main St, City7', 'Female', '+12345678907', 'sha256_7', '123-45-6795', 'State7', '404 Main St', 1617281829, 'user7', '2024-05-15T13:51:01.657574Z', 'uuid_7', 12351), - ('+12345678908', 'user8@example.com', 8.8, 1000008, 12345678908, '123-456-7897', 'Frank', 'Lopez', 'Frank Lopez', 'Sample string 8', 'abcd1241', false, '1234567812345685', 'Category8', 'City8', '505 Main St, City8', 'Male', '+12345678908', 'sha256_8', '123-45-6796', 'State8', '505 Main St', 1617281830, 'user8', '2024-05-15T13:51:01.657574Z', 'uuid_8', 12352), - ('+12345678909', 'user9@example.com', 9.9, 1000009, 12345678909, '123-456-7898', 'Grace', 'Garcia', 'Grace Garcia', 'Sample string 9', 'abcd1242', true, '1234567812345686', 'Category9', 'City9', '606 Main St, City9', 'Female', '+12345678909', 'sha256_9', '123-45-6797', 'State9', '606 Main St', 1617281831, 'user9', '2024-05-15T13:51:01.657574Z', 'uuid_9', 12353), - ('+12345678910', 'user10@example.com', 10.0, 1000010, 12345678910, '123-456-7899', 'Hank', 'Moore', 'Hank Moore', 'Sample string 10', 'abcd1243', false, '1234567812345687', 'Category10', 'City10', '707 Main St, City10', 'Male', '+12345678910', 'sha256_10', '123-45-6798', 'State10', '707 Main St', 1617281832, 'user10', '2024-05-15T13:51:01.657574Z', 'uuid_10', 12354), - ('+12345678911', 'user11@example.com', 11.1, 1000011, 12345678911, '123-456-7800', 'Ivy', 'Taylor', 'Ivy Taylor', 'Sample string 11', 'abcd1244', true, '1234567812345688', 'Category11', 'City11', '808 Main St, City11', 'Female', '+12345678911', 'sha256_11', '123-45-6799', 'State11', '808 Main St', 1617281833, 'user11', '2024-05-15T13:51:01.657574Z', 'uuid_11', 12355), - ('+12345678912', 'user12@example.com', 12.2, 1000012, 12345678912, '123-456-7801', 'Jack', 'Anderson', 'Jack Anderson', 'Sample string 12', 'abcd1245', false, '1234567812345689', 'Category12', 'City12', '909 Main St, City12', 'Male', '+12345678912', 'sha256_12', '123-45-6800', 'State12', '909 Main St', 1617281834, 'user12', '2024-05-15T13:51:01.657574Z', 'uuid_12', 12356), - ('+12345678913', 'user13@example.com', 13.3, 1000013, 12345678913, '123-456-7802', 'Kate', 'Thomas', 'Kate Thomas', 'Sample string 13', 'abcd1246', true, '1234567812345690', 'Category13', 'City13', '1010 Main St, City13', 'Female', '+12345678913', 'sha256_13', '123-45-6801', 'State13', '1010 Main St', 1617281835, 'user13', '2024-05-15T13:51:01.657574Z', 'uuid_13', 12357); + ('+12345678901', 'user1@example.com', 1.1, 1000001, 12345678901, '123-456-7890', 'John', 'Doe', 'John Doe', 'Sample string 1', 'abcd1234', true, '1234567812345678', 'Category1', 'City1', '123 Main St, City1', 'Male', 'sha256_1', '123-45-6789', 'State1', '123 Main St', 1617281823, 'user1', '2024-05-15T13:51:01.657574Z', 'uuid_1', 12345), + ('+12345678902', 'user2@example.com', 2.2, 1000002, 12345678902, '123-456-7891', 'Jane', 'Smith', 'Jane Smith', 'Sample string 2', 'abcd1235', false, '1234567812345679', 'Category2', 'City2', '456 Main St, City2', 'Female', 'sha256_2', '123-45-6790', 'State2', '456 Main St', 1617281824, 'user2', '2024-05-15T13:51:01.657574Z', 'uuid_2', 12346), + ('+12345678903', 'user3@example.com', 3.3, 1000003, 12345678903, '123-456-7892', 'Alice', 'Johnson', 'Alice Johnson', 'Sample string 3', 'abcd1236', true, '1234567812345680', 'Category3', 'City3', '789 Main St, City3', 'Female', 'sha256_3', '123-45-6791', 'State3', '789 Main St', 1617281825, 'user3', '2024-05-15T13:51:01.657574Z', 'uuid_3', 12347), + ('+12345678904', 'user4@example.com', 4.4, 1000004, 12345678904, '123-456-7893', 'Bob', 'Brown', 'Bob Brown', 'Sample string 4', 'abcd1237', false, '1234567812345681', 'Category4', 'City4', '101 Main St, City4', 'Male', 'sha256_4', '123-45-6792', 'State4', '101 Main St', 1617281826, 'user4', '2024-05-15T13:51:01.657574Z', 'uuid_4', 12348), + ('+12345678905', 'user5@example.com', 5.5, 1000005, 12345678905, '123-456-7894', 'Charlie', 'Davis', 'Charlie Davis', 'Sample string 5', 'abcd1238', true, '1234567812345682', 'Category5', 'City5', '202 Main St, City5', 'Male', 'sha256_5', '123-45-6793', 'State5', '202 Main St', 1617281827, 'user5', '2024-05-15T13:51:01.657574Z', 'uuid_5', 12349), + ('+12345678906', 'user6@example.com', 6.6, 1000006, 12345678906, '123-456-7895', 'David', 'Wilson', 'David Wilson', 'Sample string 6', 'abcd1239', false, '1234567812345683', 'Category6', 'City6', '303 Main St, City6', 'Male', 'sha256_6', '123-45-6794', 'State6', '303 Main St', 1617281828, 'user6', '2024-05-15T13:51:01.657574Z', 'uuid_6', 12350), + ('+12345678907', 'user7@example.com', 7.7, 1000007, 12345678907, '123-456-7896', 'Eve', 'Martinez', 'Eve Martinez', 'Sample string 7', 'abcd1240', true, '1234567812345684', 'Category7', 'City7', '404 Main St, City7', 'Female', 'sha256_7', '123-45-6795', 'State7', '404 Main St', 1617281829, 'user7', '2024-05-15T13:51:01.657574Z', 'uuid_7', 12351), + ('+12345678908', 'user8@example.com', 8.8, 1000008, 12345678908, '123-456-7897', 'Frank', 'Lopez', 'Frank Lopez', 'Sample string 8', 'abcd1241', false, '1234567812345685', 'Category8', 'City8', '505 Main St, City8', 'Male', 'sha256_8', '123-45-6796', 'State8', '505 Main St', 1617281830, 'user8', '2024-05-15T13:51:01.657574Z', 'uuid_8', 12352), + ('+12345678909', 'user9@example.com', 9.9, 1000009, 12345678909, '123-456-7898', 'Grace', 'Garcia', 'Grace Garcia', 'Sample string 9', 'abcd1242', true, '1234567812345686', 'Category9', 'City9', '606 Main St, City9', 'Female','sha256_9', '123-45-6797', 'State9', '606 Main St', 1617281831, 'user9', '2024-05-15T13:51:01.657574Z', 'uuid_9', 12353), + ('+12345678910', 'user10@example.com', 10.0, 1000010, 12345678910, '123-456-7899', 'Hank', 'Moore', 'Hank Moore', 'Sample string 10', 'abcd1243', false, '1234567812345687', 'Category10', 'City10', '707 Main St, City10', 'Male','sha256_10', '123-45-6798', 'State10', '707 Main St', 1617281832, 'user10', '2024-05-15T13:51:01.657574Z', 'uuid_10', 12354), + ('+12345678911', 'user11@example.com', 11.1, 1000011, 12345678911, '123-456-7800', 'Ivy', 'Taylor', 'Ivy Taylor', 'Sample string 11', 'abcd1244', true, '1234567812345688', 'Category11', 'City11', '808 Main St, City11', 'Female','sha256_11', '123-45-6799', 'State11', '808 Main St', 1617281833, 'user11', '2024-05-15T13:51:01.657574Z', 'uuid_11', 12355), + ('+12345678912', 'user12@example.com', 12.2, 1000012, 12345678912, '123-456-7801', 'Jack', 'Anderson', 'Jack Anderson', 'Sample string 12', 'abcd1245', false, '1234567812345689', 'Category12', 'City12', '909 Main St, City12', 'Male','sha256_12', '123-45-6800', 'State12', '909 Main St', 1617281834, 'user12', '2024-05-15T13:51:01.657574Z', 'uuid_12', 12356), + ('+12345678913', 'user13@example.com', 13.3, 1000013, 12345678913, '123-456-7802', 'Kate', 'Thomas', 'Kate Thomas', 'Sample string 13', 'abcd1246', true, '1234567812345690', 'Category13', 'City13', '1010 Main St, City13', 'Female','sha256_13', '123-45-6801', 'State13', '1010 Main St', 1617281835, 'user13', '2024-05-15T13:51:01.657574Z', 'uuid_13', 12357); diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go index 4327af81c0..aacaf76854 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go @@ -154,14 +154,6 @@ func GetDefaultSyncJobMappings()[]*mgmtv1alpha1.JobMapping { Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH, }, }, - { - Schema: "javascript", - Table: "transformers", - Column: "international_phone", - Transformer: &mgmtv1alpha1.JobMappingTransformer{ - Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH, - }, - }, { Schema: "javascript", Table: "transformers", diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go index 572b63a343..9b4d6e6bcd 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go @@ -32,9 +32,9 @@ func GetSyncTests() []*workflow_testdata.IntegrationTest { func getJsGeneratorJobmappings() []*mgmtv1alpha1.JobMapping { colTransformerMap := map[string]*mgmtv1alpha1.JobMappingTransformer{ - "e164_phone_number": getJavascriptTransformerConfig("return neosync.generateInternationalPhoneNumber({ min: 9, max: 15});"), + "e164_phone_number": getJavascriptTransformerConfig("return neosync.generateE164PhoneNumber({ min: 9, max: 15});"), "email": getJavascriptTransformerConfig("return neosync.generateEmail({ maxLength: 255});"), - "str": getJavascriptTransformerConfig("return neosync.generateRandomString({ min: 1, max: 50});"), + "str": getJavascriptTransformerConfig("return neosync.generateString({ min: 1, max: 50});"), "measurement": getJavascriptTransformerConfig("return neosync.generateFloat64({ min: 3.14, max: 300.10});"), "int64": getJavascriptTransformerConfig("return neosync.generateInt64({ min: 1, max: 50});"), "int64_phone_number": getJavascriptTransformerConfig("return neosync.generateInt64PhoneNumber({});"), @@ -49,7 +49,6 @@ func getJsGeneratorJobmappings() []*mgmtv1alpha1.JobMapping { "city": getJavascriptTransformerConfig("return neosync.generateCity({ maxLength: 100 });"), "full_address": getJavascriptTransformerConfig("return neosync.generateFullAddress({ maxLength: 100 });"), "gender": getJavascriptTransformerConfig("return neosync.generateGender({});"), - "international_phone": getJavascriptTransformerConfig("return neosync.generateInternationalPhoneNumber({ min: 9, max: 14});"), "sha256": getJavascriptTransformerConfig("return neosync.generateSHA256Hash({});"), "ssn": getJavascriptTransformerConfig("return neosync.generateSSN({});"), "state": getJavascriptTransformerConfig("return neosync.generateState({});"), @@ -85,7 +84,7 @@ func getJsTransformerJobmappings() []*mgmtv1alpha1.JobMapping { "measurement": getJavascriptTransformerConfig("return neosync.transformFloat64(value, { randomizationRangeMin: 3.14, randomizationRangeMax: 300.10});"), "int64": getJavascriptTransformerConfig("return neosync.transformInt64(value, { randomizationRangeMin: 1, randomizationRangeMax: 300});"), "int64_phone_number": getJavascriptTransformerConfig("return neosync.transformInt64PhoneNumber(value, { preserveLength: true});"), - "string_phone_number": getJavascriptTransformerConfig("return neosync.transformStringPhoneNumber(value, { preserveLength: true, maxLength: 200});"), + "string_phone_number": getJavascriptTransformerConfig("return neosync.transformPhoneNumber(value, { preserveLength: true, maxLength: 200});"), "first_name": getJavascriptTransformerConfig("return neosync.transformFirstName(value, { preserveLength: true, maxLength: 25});"), "last_name": getJavascriptTransformerConfig("return neosync.transformLastName(value, { preserveLength: true, maxLength: 25});"), "full_name": getJavascriptTransformerConfig("return neosync.transformFullName(value, { preserveLength: true, maxLength: 25});"), From 6da57081d7756365634ca699a6f65adbf13266bb Mon Sep 17 00:00:00 2001 From: Alisha Date: Wed, 16 Oct 2024 15:36:02 -0700 Subject: [PATCH 09/32] revert breaking changes to transformer names --- .../gen-javascript-transformer.md | 152 +++++++++--------- .../apps/web/@types/neosync-transformers.d.ts | 80 ++++----- ...en_generate_international_phone_number.go} | 24 +-- ...tring.go => gen_generate_random_string.go} | 24 +-- .../transformers/gen_neosync_transformers.go | 6 +- ...o => gen_transform_string_phone_number.go} | 24 +-- ...=> generate_international_phone_number.go} | 6 +- ...nerate_international_phone_number_test.go} | 0 ...te_string.go => generate_random_string.go} | 6 +- ...test.go => generate_random_string_test.go} | 0 ...er.go => transform_string_phone_number.go} | 6 +- ... => transform_string_phone_number_test.go} | 0 .../transformers/transformer_initializer.go | 12 +- .../javascript-transformers/create.sql | 1 + .../javascript-transformers/insert.sql | 28 ++-- .../javascript-transformers/job_mappings.go | 8 + .../testdata/javascript-transformers/tests.go | 7 +- 17 files changed, 197 insertions(+), 187 deletions(-) rename worker/pkg/benthos/transformers/{gen_generate_e164_phone_number.go => gen_generate_international_phone_number.go} (66%) rename worker/pkg/benthos/transformers/{gen_generate_string.go => gen_generate_random_string.go} (71%) rename worker/pkg/benthos/transformers/{gen_transform_phone_number.go => gen_transform_string_phone_number.go} (72%) rename worker/pkg/benthos/transformers/{generate_e164_phone_number.go => generate_international_phone_number.go} (91%) rename worker/pkg/benthos/transformers/{generate_e164_phone_number_test.go => generate_international_phone_number_test.go} (100%) rename worker/pkg/benthos/transformers/{generate_string.go => generate_random_string.go} (90%) rename worker/pkg/benthos/transformers/{generate_string_test.go => generate_random_string_test.go} (100%) rename worker/pkg/benthos/transformers/{transform_phone_number.go => transform_string_phone_number.go} (93%) rename worker/pkg/benthos/transformers/{transform_phone_number_test.go => transform_string_phone_number_test.go} (100%) diff --git a/docs/docs/transformers/gen-javascript-transformer.md b/docs/docs/transformers/gen-javascript-transformer.md index dbe5e0346b..fb6aea2703 100644 --- a/docs/docs/transformers/gen-javascript-transformer.md +++ b/docs/docs/transformers/gen-javascript-transformer.md @@ -358,12 +358,12 @@ const newValue = neosync.transformLastName(value, { -### transformPhoneNumber +### transformString -Anonymizes and transforms an existing phone number that is typed as a string. +Anonymizes and transforms an existing string value. **Parameters** @@ -376,15 +376,17 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. +| minLength | int64 | 1 | false | Specifies the minimum length of the transformed value. +| maxLength | int64 | 100 | false | Specifies the maximum length of the transformed value. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.transformPhoneNumber(value, { +const newValue = neosync.transformString(value, { preserveLength: false, + minLength: 1, maxLength: 100, seed: 1, }); @@ -394,12 +396,12 @@ const newValue = neosync.transformPhoneNumber(value, { -### transformString +### transformStringPhoneNumber -Anonymizes and transforms an existing string value. +Anonymizes and transforms an existing phone number that is typed as a string. **Parameters** @@ -412,17 +414,15 @@ Description: Value that will be transformed | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | | preserveLength | bool | false | false | Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. -| minLength | int64 | 1 | false | Specifies the minimum length of the transformed value. -| maxLength | int64 | 100 | false | Specifies the maximum length of the transformed value. +| maxLength | int64 | 100 | false | Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.transformString(value, { +const newValue = neosync.transformStringPhoneNumber(value, { preserveLength: false, - minLength: 1, maxLength: 100, seed: 1, }); @@ -593,39 +593,6 @@ const newValue = neosync.generateCountry({
- - -### generateE164PhoneNumber - -Generates a new random international phone number including the + sign and no hyphens. - -**Parameters** - -**Config** - -| Field | Type | Default | Required | Description | -| -------- | ---- | ------- | -------- | ----------- | -| min | int64 | 9 | false | Specifies the minimum value for the generated phone number. -| max | int64 | 15 | false | Specifies the maximum value for the generated phone number. -| seed | int64 | | false | An optional seed value used to generate deterministic outputs. -
- -**Example** - -```javascript - -const newValue = neosync.generateE164PhoneNumber({ - min: 9, - max: 15, - seed: 1, -}); - -``` -
- - @@ -888,6 +855,39 @@ const newValue = neosync.generateInt64PhoneNumber({
+ + +### generateInternationalPhoneNumber + +Generates a new random international phone number including the + sign and no hyphens. + +**Parameters** + +**Config** + +| Field | Type | Default | Required | Description | +| -------- | ---- | ------- | -------- | ----------- | +| min | int64 | 9 | false | Specifies the minimum value for the generated phone number. +| max | int64 | 15 | false | Specifies the maximum value for the generated phone number. +| seed | int64 | | false | An optional seed value used to generate deterministic outputs. +
+ +**Example** + +```javascript + +const newValue = neosync.generateInternationalPhoneNumber({ + min: 9, + max: 15, + seed: 1, +}); + +``` +
+ + @@ -920,12 +920,12 @@ const newValue = neosync.generateLastName({ -### generateSHA256Hash +### generateRandomString -Generates a random SHA256 hash and returns it as a string. +Generates a random string of alphanumeric characters.. **Parameters** @@ -933,25 +933,32 @@ Generates a random SHA256 hash and returns it as a string. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | +| min | int64 | 1 | false | Specifies the minimum length for the generated string. +| max | int64 | 100 | false | Specifies the maximum length for the generated string. +| seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.generateSHA256Hash({}); +const newValue = neosync.generateRandomString({ + min: 1, + max: 100, + seed: 1, +}); ```
-### generateSSN +### generateSHA256Hash -Generates a random social security numbers including the hyphens in the format xxx-xx-xxxx. +Generates a random SHA256 hash and returns it as a string. **Parameters** @@ -959,28 +966,25 @@ Generates a random social security numbers including the hyphens in the format x | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| seed | int64 | | false | An optional seed value used to generate deterministic outputs.
**Example** ```javascript -const newValue = neosync.generateSSN({ - seed: 1, -}); +const newValue = neosync.generateSHA256Hash({}); ```
-### generateState +### generateSSN -Randomly selects a US state and by default, returns it as a 2-letter state code. +Generates a random social security numbers including the hyphens in the format xxx-xx-xxxx. **Parameters** @@ -988,7 +992,6 @@ Randomly selects a US state and by default, returns it as a 2-letter state code. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| generateFullName | bool | false | false | If true returns the full state name instead of the two character state code. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -996,8 +999,7 @@ Randomly selects a US state and by default, returns it as a 2-letter state code. ```javascript -const newValue = neosync.generateState({ - generateFullName: false, +const newValue = neosync.generateSSN({ seed: 1, }); @@ -1006,12 +1008,12 @@ const newValue = neosync.generateState({ -### generateStreetAddress +### generateState -Randomly generates a street address. +Randomly selects a US state and by default, returns it as a 2-letter state code. **Parameters** @@ -1019,7 +1021,7 @@ Randomly generates a street address. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. +| generateFullName | bool | false | false | If true returns the full state name instead of the two character state code. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1027,8 +1029,8 @@ Randomly generates a street address. ```javascript -const newValue = neosync.generateStreetAddress({ - maxLength: 100, +const newValue = neosync.generateState({ + generateFullName: false, seed: 1, }); @@ -1037,12 +1039,12 @@ const newValue = neosync.generateStreetAddress({ -### generateString +### generateStreetAddress -Generates a random string of alphanumeric characters.. +Randomly generates a street address. **Parameters** @@ -1050,8 +1052,7 @@ Generates a random string of alphanumeric characters.. | Field | Type | Default | Required | Description | | -------- | ---- | ------- | -------- | ----------- | -| min | int64 | 1 | false | Specifies the minimum length for the generated string. -| max | int64 | 100 | false | Specifies the maximum length for the generated string. +| maxLength | int64 | 100 | false | Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. | seed | int64 | | false | An optional seed value used to generate deterministic outputs.
@@ -1059,9 +1060,8 @@ Generates a random string of alphanumeric characters.. ```javascript -const newValue = neosync.generateString({ - min: 1, - max: 100, +const newValue = neosync.generateStreetAddress({ + maxLength: 100, seed: 1, }); diff --git a/frontend/apps/web/@types/neosync-transformers.d.ts b/frontend/apps/web/@types/neosync-transformers.d.ts index bde04a5b04..3c2c4ea0a4 100644 --- a/frontend/apps/web/@types/neosync-transformers.d.ts +++ b/frontend/apps/web/@types/neosync-transformers.d.ts @@ -155,36 +155,36 @@ declare namespace neosync { declare function transformLastName(value: any, options: TransformLastNameOptions): any; - export interface TransformPhoneNumberOptions { + export interface TransformStringOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ preserveLength?: boolean; - /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ + /** Specifies the minimum length of the transformed value. */ + minLength?: number; + /** Specifies the maximum length of the transformed value. */ maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } /** - * Anonymizes and transforms an existing phone number that is typed as a string. + * Anonymizes and transforms an existing string value. */ - declare function transformPhoneNumber(value: any, options: TransformPhoneNumberOptions): any; + declare function transformString(value: any, options: TransformStringOptions): any; - export interface TransformStringOptions { + export interface TransformStringPhoneNumberOptions { /** Whether the original length of the input data should be preserved during transformation. If set to true, the transformation logic will ensure that the output data has the same length as the input data. */ preserveLength?: boolean; - /** Specifies the minimum length of the transformed value. */ - minLength?: number; - /** Specifies the maximum length of the transformed value. */ + /** Specifies the maximum length for the transformed data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; /** An optional seed value used to generate deterministic outputs. */ seed?: number; } /** - * Anonymizes and transforms an existing string value. + * Anonymizes and transforms an existing phone number that is typed as a string. */ - declare function transformString(value: any, options: TransformStringOptions): any; + declare function transformStringPhoneNumber(value: any, options: TransformStringPhoneNumberOptions): any; @@ -256,21 +256,6 @@ declare namespace neosync { declare function generateCountry(options: GenerateCountryOptions): any; - export interface GenerateE164PhoneNumberOptions { - /** Specifies the minimum value for the generated phone number. */ - min?: number; - /** Specifies the maximum value for the generated phone number. */ - max?: number; - /** An optional seed value used to generate deterministic outputs. */ - seed?: number; - } - - /** - * Generates a new random international phone number including the + sign and no hyphens. - */ - declare function generateE164PhoneNumber(options: GenerateE164PhoneNumberOptions): any; - - export interface GenerateEmailOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; @@ -389,6 +374,21 @@ declare namespace neosync { declare function generateInt64PhoneNumber(options: GenerateInt64PhoneNumberOptions): any; + export interface GenerateInternationalPhoneNumberOptions { + /** Specifies the minimum value for the generated phone number. */ + min?: number; + /** Specifies the maximum value for the generated phone number. */ + max?: number; + /** An optional seed value used to generate deterministic outputs. */ + seed?: number; + } + + /** + * Generates a new random international phone number including the + sign and no hyphens. + */ + declare function generateInternationalPhoneNumber(options: GenerateInternationalPhoneNumberOptions): any; + + export interface GenerateLastNameOptions { /** Specifies the maximum length for the generated data. This field ensures that the output does not exceed a certain number of characters. */ maxLength?: number; @@ -402,6 +402,21 @@ declare namespace neosync { declare function generateLastName(options: GenerateLastNameOptions): any; + export interface GenerateRandomStringOptions { + /** Specifies the minimum length for the generated string. */ + min?: number; + /** Specifies the maximum length for the generated string. */ + max?: number; + /** An optional seed value used to generate deterministic outputs. */ + seed?: number; + } + + /** + * Generates a random string of alphanumeric characters.. + */ + declare function generateRandomString(options: GenerateRandomStringOptions): any; + + export interface GenerateSHA256HashOptions { } @@ -448,21 +463,6 @@ declare namespace neosync { declare function generateStreetAddress(options: GenerateStreetAddressOptions): any; - export interface GenerateStringOptions { - /** Specifies the minimum length for the generated string. */ - min?: number; - /** Specifies the maximum length for the generated string. */ - max?: number; - /** An optional seed value used to generate deterministic outputs. */ - seed?: number; - } - - /** - * Generates a random string of alphanumeric characters.. - */ - declare function generateString(options: GenerateStringOptions): any; - - export interface GenerateStringPhoneNumberOptions { /** Specifies the minimum length for the generated phone number. */ min?: number; diff --git a/worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go similarity index 66% rename from worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go rename to worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index 3fcdbd5575..693b6eee35 100644 --- a/worker/pkg/benthos/transformers/gen_generate_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: generate_e164_phone_number.go +// source: generate_international_phone_number.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type GenerateE164PhoneNumber struct{} +type GenerateInternationalPhoneNumber struct{} -type GenerateE164PhoneNumberOpts struct { +type GenerateInternationalPhoneNumberOpts struct { randomizer rng.Rand min int64 max int64 } -func NewGenerateE164PhoneNumber() *GenerateE164PhoneNumber { - return &GenerateE164PhoneNumber{} +func NewGenerateInternationalPhoneNumber() *GenerateInternationalPhoneNumber { + return &GenerateInternationalPhoneNumber{} } -func NewGenerateE164PhoneNumberOpts( +func NewGenerateInternationalPhoneNumberOpts( minArg *int64, maxArg *int64, seedArg *int64, -) (*GenerateE164PhoneNumberOpts, error) { +) (*GenerateInternationalPhoneNumberOpts, error) { min := int64(9) if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { min = *minArg @@ -44,23 +44,23 @@ func NewGenerateE164PhoneNumberOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &GenerateE164PhoneNumberOpts{ + return &GenerateInternationalPhoneNumberOpts{ min: min, max: max, randomizer: rng.New(seed), }, nil } -func (t *GenerateE164PhoneNumber) GetJsTemplateData() (*TemplateData, error) { +func (t *GenerateInternationalPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "generateE164PhoneNumber", + Name: "generateInternationalPhoneNumber", Description: "Generates a new random international phone number including the + sign and no hyphens.", Example: "", }, nil } -func (t *GenerateE164PhoneNumber) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &GenerateE164PhoneNumberOpts{} +func (t *GenerateInternationalPhoneNumber) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &GenerateInternationalPhoneNumberOpts{} min, ok := opts["min"].(int64) if !ok { diff --git a/worker/pkg/benthos/transformers/gen_generate_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go similarity index 71% rename from worker/pkg/benthos/transformers/gen_generate_string.go rename to worker/pkg/benthos/transformers/gen_generate_random_string.go index 506c2b0566..9971041761 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: generate_string.go +// source: generate_random_string.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type GenerateString struct{} +type GenerateRandomString struct{} -type GenerateStringOpts struct { +type GenerateRandomStringOpts struct { randomizer rng.Rand min int64 max int64 } -func NewGenerateString() *GenerateString { - return &GenerateString{} +func NewGenerateRandomString() *GenerateRandomString { + return &GenerateRandomString{} } -func NewGenerateStringOpts( +func NewGenerateRandomStringOpts( minArg *int64, maxArg *int64, seedArg *int64, -) (*GenerateStringOpts, error) { +) (*GenerateRandomStringOpts, error) { min := int64(1) if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { min = *minArg @@ -44,23 +44,23 @@ func NewGenerateStringOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &GenerateStringOpts{ + return &GenerateRandomStringOpts{ min: min, max: max, randomizer: rng.New(seed), }, nil } -func (t *GenerateString) GetJsTemplateData() (*TemplateData, error) { +func (t *GenerateRandomString) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "generateString", + Name: "generateRandomString", Description: "Generates a random string of alphanumeric characters..", Example: "", }, nil } -func (t *GenerateString) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &GenerateStringOpts{} +func (t *GenerateRandomString) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &GenerateRandomStringOpts{} min, ok := opts["min"].(int64) if !ok { diff --git a/worker/pkg/benthos/transformers/gen_neosync_transformers.go b/worker/pkg/benthos/transformers/gen_neosync_transformers.go index f9f5e67746..6a256fe070 100644 --- a/worker/pkg/benthos/transformers/gen_neosync_transformers.go +++ b/worker/pkg/benthos/transformers/gen_neosync_transformers.go @@ -14,8 +14,8 @@ func GetNeosyncTransformers() []NeosyncTransformer { NewTransformInt64(), NewTransformInt64PhoneNumber(), NewTransformLastName(), - NewTransformPhoneNumber(), NewTransformString(), + NewTransformStringPhoneNumber(), } } @@ -26,7 +26,6 @@ func GetNeosyncGenerators() []NeosyncGenerator { NewGenerateCategorical(), NewGenerateCity(), NewGenerateCountry(), - NewGenerateE164PhoneNumber(), NewGenerateEmail(), NewGenerateFirstName(), NewGenerateFloat64(), @@ -35,12 +34,13 @@ func GetNeosyncGenerators() []NeosyncGenerator { NewGenerateGender(), NewGenerateInt64(), NewGenerateInt64PhoneNumber(), + NewGenerateInternationalPhoneNumber(), NewGenerateLastName(), + NewGenerateRandomString(), NewGenerateSHA256Hash(), NewGenerateSSN(), NewGenerateState(), NewGenerateStreetAddress(), - NewGenerateString(), NewGenerateStringPhoneNumber(), NewGenerateUnixTimestamp(), NewGenerateUsername(), diff --git a/worker/pkg/benthos/transformers/gen_transform_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go similarity index 72% rename from worker/pkg/benthos/transformers/gen_transform_phone_number.go rename to worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 44f86d0b29..726ff6a619 100644 --- a/worker/pkg/benthos/transformers/gen_transform_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -1,6 +1,6 @@ // Code generated by Neosync neosync_transformer_generator.go. DO NOT EDIT. -// source: transform_phone_number.go +// source: transform_string_phone_number.go package transformers @@ -11,24 +11,24 @@ import ( transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" ) -type TransformPhoneNumber struct{} +type TransformStringPhoneNumber struct{} -type TransformPhoneNumberOpts struct { +type TransformStringPhoneNumberOpts struct { randomizer rng.Rand preserveLength bool maxLength int64 } -func NewTransformPhoneNumber() *TransformPhoneNumber { - return &TransformPhoneNumber{} +func NewTransformStringPhoneNumber() *TransformStringPhoneNumber { + return &TransformStringPhoneNumber{} } -func NewTransformPhoneNumberOpts( +func NewTransformStringPhoneNumberOpts( preserveLengthArg *bool, maxLengthArg *int64, seedArg *int64, -) (*TransformPhoneNumberOpts, error) { +) (*TransformStringPhoneNumberOpts, error) { preserveLength := bool(false) if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { preserveLength = *preserveLengthArg @@ -44,23 +44,23 @@ func NewTransformPhoneNumberOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - return &TransformPhoneNumberOpts{ + return &TransformStringPhoneNumberOpts{ preserveLength: preserveLength, maxLength: maxLength, randomizer: rng.New(seed), }, nil } -func (t *TransformPhoneNumber) GetJsTemplateData() (*TemplateData, error) { +func (t *TransformStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ - Name: "transformPhoneNumber", + Name: "transformStringPhoneNumber", Description: "Anonymizes and transforms an existing phone number that is typed as a string.", Example: "", }, nil } -func (t *TransformPhoneNumber) ParseOptions(opts map[string]any) (any, error) { - transformerOpts := &TransformPhoneNumberOpts{} +func (t *TransformStringPhoneNumber) ParseOptions(opts map[string]any) (any, error) { + transformerOpts := &TransformStringPhoneNumberOpts{} preserveLength, ok := opts["preserveLength"].(bool) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_e164_phone_number.go b/worker/pkg/benthos/transformers/generate_international_phone_number.go similarity index 91% rename from worker/pkg/benthos/transformers/generate_e164_phone_number.go rename to worker/pkg/benthos/transformers/generate_international_phone_number.go index 0fd0ff4a48..715f48cc17 100644 --- a/worker/pkg/benthos/transformers/generate_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_international_phone_number.go @@ -10,7 +10,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:generate:generateE164PhoneNumber +// +neosyncTransformerBuilder:generate:generateInternationalPhoneNumber func init() { spec := bloblang.NewPluginSpec(). @@ -56,8 +56,8 @@ func init() { } } -func (t *GenerateE164PhoneNumber) Generate(opts any) (any, error) { - parsedOpts, ok := opts.(*GenerateE164PhoneNumberOpts) +func (t *GenerateInternationalPhoneNumber) Generate(opts any) (any, error) { + parsedOpts, ok := opts.(*GenerateInternationalPhoneNumberOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/generate_e164_phone_number_test.go b/worker/pkg/benthos/transformers/generate_international_phone_number_test.go similarity index 100% rename from worker/pkg/benthos/transformers/generate_e164_phone_number_test.go rename to worker/pkg/benthos/transformers/generate_international_phone_number_test.go diff --git a/worker/pkg/benthos/transformers/generate_string.go b/worker/pkg/benthos/transformers/generate_random_string.go similarity index 90% rename from worker/pkg/benthos/transformers/generate_string.go rename to worker/pkg/benthos/transformers/generate_random_string.go index a632490aa1..69b1bdad36 100644 --- a/worker/pkg/benthos/transformers/generate_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -8,7 +8,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:generate:generateString +// +neosyncTransformerBuilder:generate:generateRandomString func init() { spec := bloblang.NewPluginSpec(). @@ -54,8 +54,8 @@ func init() { } } -func (t *GenerateString) Generate(opts any) (any, error) { - parsedOpts, ok := opts.(*GenerateStringOpts) +func (t *GenerateRandomString) Generate(opts any) (any, error) { + parsedOpts, ok := opts.(*GenerateRandomStringOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/generate_string_test.go b/worker/pkg/benthos/transformers/generate_random_string_test.go similarity index 100% rename from worker/pkg/benthos/transformers/generate_string_test.go rename to worker/pkg/benthos/transformers/generate_random_string_test.go diff --git a/worker/pkg/benthos/transformers/transform_phone_number.go b/worker/pkg/benthos/transformers/transform_string_phone_number.go similarity index 93% rename from worker/pkg/benthos/transformers/transform_phone_number.go rename to worker/pkg/benthos/transformers/transform_string_phone_number.go index 0e2a5fbe74..e587089f12 100644 --- a/worker/pkg/benthos/transformers/transform_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_string_phone_number.go @@ -9,7 +9,7 @@ import ( "github.com/warpstreamlabs/bento/public/bloblang" ) -// +neosyncTransformerBuilder:transform:transformPhoneNumber +// +neosyncTransformerBuilder:transform:transformStringPhoneNumber func init() { spec := bloblang.NewPluginSpec(). @@ -61,8 +61,8 @@ func init() { } } -func (t *TransformPhoneNumber) Transform(value, opts any) (any, error) { - parsedOpts, ok := opts.(*TransformPhoneNumberOpts) +func (t *TransformStringPhoneNumber) Transform(value, opts any) (any, error) { + parsedOpts, ok := opts.(*TransformStringPhoneNumberOpts) if !ok { return nil, fmt.Errorf("invalid parsed opts: %T", opts) } diff --git a/worker/pkg/benthos/transformers/transform_phone_number_test.go b/worker/pkg/benthos/transformers/transform_string_phone_number_test.go similarity index 100% rename from worker/pkg/benthos/transformers/transform_phone_number_test.go rename to worker/pkg/benthos/transformers/transform_string_phone_number_test.go diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index c4576a4e6f..4f46c13e43 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -204,11 +204,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: config := transformerConfig.GetGenerateE164PhoneNumberConfig() - opts, err := NewGenerateE164PhoneNumberOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateInternationalPhoneNumberOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } - generate := NewGenerateE164PhoneNumber().Generate + generate := NewGenerateInternationalPhoneNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -402,11 +402,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateStringOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateRandomStringOpts(&config.Min, &config.Max, nil) if err != nil { return nil, err } - generate := NewGenerateString().Generate + generate := NewGenerateRandomString().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -558,11 +558,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) + opts, err := NewTransformStringPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) if err != nil { return nil, err } - transform := NewTransformPhoneNumber().Transform + transform := NewTransformStringPhoneNumber().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql index 968a835601..c78345c920 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/create.sql @@ -20,6 +20,7 @@ CREATE TABLE transformers ( city VARCHAR(255), full_address VARCHAR(255), gender VARCHAR(255), + international_phone VARCHAR(255), sha256 VARCHAR(255), ssn VARCHAR(255), state VARCHAR(255), diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql index a0a6b498e8..b823a23b99 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/insert.sql @@ -2,19 +2,19 @@ SET search_path TO javascript; INSERT INTO transformers ( e164_phone_number, email, measurement, int64, int64_phone_number, string_phone_number, first_name, last_name, full_name, str, character_scramble, bool, card_number, categorical, - city, full_address, gender, sha256, ssn, state, street_address, + city, full_address, gender, international_phone, sha256, ssn, state, street_address, unix_time, username, utc_timestamp, uuid, zipcode ) VALUES - ('+12345678901', 'user1@example.com', 1.1, 1000001, 12345678901, '123-456-7890', 'John', 'Doe', 'John Doe', 'Sample string 1', 'abcd1234', true, '1234567812345678', 'Category1', 'City1', '123 Main St, City1', 'Male', 'sha256_1', '123-45-6789', 'State1', '123 Main St', 1617281823, 'user1', '2024-05-15T13:51:01.657574Z', 'uuid_1', 12345), - ('+12345678902', 'user2@example.com', 2.2, 1000002, 12345678902, '123-456-7891', 'Jane', 'Smith', 'Jane Smith', 'Sample string 2', 'abcd1235', false, '1234567812345679', 'Category2', 'City2', '456 Main St, City2', 'Female', 'sha256_2', '123-45-6790', 'State2', '456 Main St', 1617281824, 'user2', '2024-05-15T13:51:01.657574Z', 'uuid_2', 12346), - ('+12345678903', 'user3@example.com', 3.3, 1000003, 12345678903, '123-456-7892', 'Alice', 'Johnson', 'Alice Johnson', 'Sample string 3', 'abcd1236', true, '1234567812345680', 'Category3', 'City3', '789 Main St, City3', 'Female', 'sha256_3', '123-45-6791', 'State3', '789 Main St', 1617281825, 'user3', '2024-05-15T13:51:01.657574Z', 'uuid_3', 12347), - ('+12345678904', 'user4@example.com', 4.4, 1000004, 12345678904, '123-456-7893', 'Bob', 'Brown', 'Bob Brown', 'Sample string 4', 'abcd1237', false, '1234567812345681', 'Category4', 'City4', '101 Main St, City4', 'Male', 'sha256_4', '123-45-6792', 'State4', '101 Main St', 1617281826, 'user4', '2024-05-15T13:51:01.657574Z', 'uuid_4', 12348), - ('+12345678905', 'user5@example.com', 5.5, 1000005, 12345678905, '123-456-7894', 'Charlie', 'Davis', 'Charlie Davis', 'Sample string 5', 'abcd1238', true, '1234567812345682', 'Category5', 'City5', '202 Main St, City5', 'Male', 'sha256_5', '123-45-6793', 'State5', '202 Main St', 1617281827, 'user5', '2024-05-15T13:51:01.657574Z', 'uuid_5', 12349), - ('+12345678906', 'user6@example.com', 6.6, 1000006, 12345678906, '123-456-7895', 'David', 'Wilson', 'David Wilson', 'Sample string 6', 'abcd1239', false, '1234567812345683', 'Category6', 'City6', '303 Main St, City6', 'Male', 'sha256_6', '123-45-6794', 'State6', '303 Main St', 1617281828, 'user6', '2024-05-15T13:51:01.657574Z', 'uuid_6', 12350), - ('+12345678907', 'user7@example.com', 7.7, 1000007, 12345678907, '123-456-7896', 'Eve', 'Martinez', 'Eve Martinez', 'Sample string 7', 'abcd1240', true, '1234567812345684', 'Category7', 'City7', '404 Main St, City7', 'Female', 'sha256_7', '123-45-6795', 'State7', '404 Main St', 1617281829, 'user7', '2024-05-15T13:51:01.657574Z', 'uuid_7', 12351), - ('+12345678908', 'user8@example.com', 8.8, 1000008, 12345678908, '123-456-7897', 'Frank', 'Lopez', 'Frank Lopez', 'Sample string 8', 'abcd1241', false, '1234567812345685', 'Category8', 'City8', '505 Main St, City8', 'Male', 'sha256_8', '123-45-6796', 'State8', '505 Main St', 1617281830, 'user8', '2024-05-15T13:51:01.657574Z', 'uuid_8', 12352), - ('+12345678909', 'user9@example.com', 9.9, 1000009, 12345678909, '123-456-7898', 'Grace', 'Garcia', 'Grace Garcia', 'Sample string 9', 'abcd1242', true, '1234567812345686', 'Category9', 'City9', '606 Main St, City9', 'Female','sha256_9', '123-45-6797', 'State9', '606 Main St', 1617281831, 'user9', '2024-05-15T13:51:01.657574Z', 'uuid_9', 12353), - ('+12345678910', 'user10@example.com', 10.0, 1000010, 12345678910, '123-456-7899', 'Hank', 'Moore', 'Hank Moore', 'Sample string 10', 'abcd1243', false, '1234567812345687', 'Category10', 'City10', '707 Main St, City10', 'Male','sha256_10', '123-45-6798', 'State10', '707 Main St', 1617281832, 'user10', '2024-05-15T13:51:01.657574Z', 'uuid_10', 12354), - ('+12345678911', 'user11@example.com', 11.1, 1000011, 12345678911, '123-456-7800', 'Ivy', 'Taylor', 'Ivy Taylor', 'Sample string 11', 'abcd1244', true, '1234567812345688', 'Category11', 'City11', '808 Main St, City11', 'Female','sha256_11', '123-45-6799', 'State11', '808 Main St', 1617281833, 'user11', '2024-05-15T13:51:01.657574Z', 'uuid_11', 12355), - ('+12345678912', 'user12@example.com', 12.2, 1000012, 12345678912, '123-456-7801', 'Jack', 'Anderson', 'Jack Anderson', 'Sample string 12', 'abcd1245', false, '1234567812345689', 'Category12', 'City12', '909 Main St, City12', 'Male','sha256_12', '123-45-6800', 'State12', '909 Main St', 1617281834, 'user12', '2024-05-15T13:51:01.657574Z', 'uuid_12', 12356), - ('+12345678913', 'user13@example.com', 13.3, 1000013, 12345678913, '123-456-7802', 'Kate', 'Thomas', 'Kate Thomas', 'Sample string 13', 'abcd1246', true, '1234567812345690', 'Category13', 'City13', '1010 Main St, City13', 'Female','sha256_13', '123-45-6801', 'State13', '1010 Main St', 1617281835, 'user13', '2024-05-15T13:51:01.657574Z', 'uuid_13', 12357); + ('+12345678901', 'user1@example.com', 1.1, 1000001, 12345678901, '123-456-7890', 'John', 'Doe', 'John Doe', 'Sample string 1', 'abcd1234', true, '1234567812345678', 'Category1', 'City1', '123 Main St, City1', 'Male', '+12345678901', 'sha256_1', '123-45-6789', 'State1', '123 Main St', 1617281823, 'user1', '2024-05-15T13:51:01.657574Z', 'uuid_1', 12345), + ('+12345678902', 'user2@example.com', 2.2, 1000002, 12345678902, '123-456-7891', 'Jane', 'Smith', 'Jane Smith', 'Sample string 2', 'abcd1235', false, '1234567812345679', 'Category2', 'City2', '456 Main St, City2', 'Female', '+12345678902', 'sha256_2', '123-45-6790', 'State2', '456 Main St', 1617281824, 'user2', '2024-05-15T13:51:01.657574Z', 'uuid_2', 12346), + ('+12345678903', 'user3@example.com', 3.3, 1000003, 12345678903, '123-456-7892', 'Alice', 'Johnson', 'Alice Johnson', 'Sample string 3', 'abcd1236', true, '1234567812345680', 'Category3', 'City3', '789 Main St, City3', 'Female', '+12345678903', 'sha256_3', '123-45-6791', 'State3', '789 Main St', 1617281825, 'user3', '2024-05-15T13:51:01.657574Z', 'uuid_3', 12347), + ('+12345678904', 'user4@example.com', 4.4, 1000004, 12345678904, '123-456-7893', 'Bob', 'Brown', 'Bob Brown', 'Sample string 4', 'abcd1237', false, '1234567812345681', 'Category4', 'City4', '101 Main St, City4', 'Male', '+12345678904', 'sha256_4', '123-45-6792', 'State4', '101 Main St', 1617281826, 'user4', '2024-05-15T13:51:01.657574Z', 'uuid_4', 12348), + ('+12345678905', 'user5@example.com', 5.5, 1000005, 12345678905, '123-456-7894', 'Charlie', 'Davis', 'Charlie Davis', 'Sample string 5', 'abcd1238', true, '1234567812345682', 'Category5', 'City5', '202 Main St, City5', 'Male', '+12345678905', 'sha256_5', '123-45-6793', 'State5', '202 Main St', 1617281827, 'user5', '2024-05-15T13:51:01.657574Z', 'uuid_5', 12349), + ('+12345678906', 'user6@example.com', 6.6, 1000006, 12345678906, '123-456-7895', 'David', 'Wilson', 'David Wilson', 'Sample string 6', 'abcd1239', false, '1234567812345683', 'Category6', 'City6', '303 Main St, City6', 'Male', '+12345678906', 'sha256_6', '123-45-6794', 'State6', '303 Main St', 1617281828, 'user6', '2024-05-15T13:51:01.657574Z', 'uuid_6', 12350), + ('+12345678907', 'user7@example.com', 7.7, 1000007, 12345678907, '123-456-7896', 'Eve', 'Martinez', 'Eve Martinez', 'Sample string 7', 'abcd1240', true, '1234567812345684', 'Category7', 'City7', '404 Main St, City7', 'Female', '+12345678907', 'sha256_7', '123-45-6795', 'State7', '404 Main St', 1617281829, 'user7', '2024-05-15T13:51:01.657574Z', 'uuid_7', 12351), + ('+12345678908', 'user8@example.com', 8.8, 1000008, 12345678908, '123-456-7897', 'Frank', 'Lopez', 'Frank Lopez', 'Sample string 8', 'abcd1241', false, '1234567812345685', 'Category8', 'City8', '505 Main St, City8', 'Male', '+12345678908', 'sha256_8', '123-45-6796', 'State8', '505 Main St', 1617281830, 'user8', '2024-05-15T13:51:01.657574Z', 'uuid_8', 12352), + ('+12345678909', 'user9@example.com', 9.9, 1000009, 12345678909, '123-456-7898', 'Grace', 'Garcia', 'Grace Garcia', 'Sample string 9', 'abcd1242', true, '1234567812345686', 'Category9', 'City9', '606 Main St, City9', 'Female', '+12345678909', 'sha256_9', '123-45-6797', 'State9', '606 Main St', 1617281831, 'user9', '2024-05-15T13:51:01.657574Z', 'uuid_9', 12353), + ('+12345678910', 'user10@example.com', 10.0, 1000010, 12345678910, '123-456-7899', 'Hank', 'Moore', 'Hank Moore', 'Sample string 10', 'abcd1243', false, '1234567812345687', 'Category10', 'City10', '707 Main St, City10', 'Male', '+12345678910', 'sha256_10', '123-45-6798', 'State10', '707 Main St', 1617281832, 'user10', '2024-05-15T13:51:01.657574Z', 'uuid_10', 12354), + ('+12345678911', 'user11@example.com', 11.1, 1000011, 12345678911, '123-456-7800', 'Ivy', 'Taylor', 'Ivy Taylor', 'Sample string 11', 'abcd1244', true, '1234567812345688', 'Category11', 'City11', '808 Main St, City11', 'Female', '+12345678911', 'sha256_11', '123-45-6799', 'State11', '808 Main St', 1617281833, 'user11', '2024-05-15T13:51:01.657574Z', 'uuid_11', 12355), + ('+12345678912', 'user12@example.com', 12.2, 1000012, 12345678912, '123-456-7801', 'Jack', 'Anderson', 'Jack Anderson', 'Sample string 12', 'abcd1245', false, '1234567812345689', 'Category12', 'City12', '909 Main St, City12', 'Male', '+12345678912', 'sha256_12', '123-45-6800', 'State12', '909 Main St', 1617281834, 'user12', '2024-05-15T13:51:01.657574Z', 'uuid_12', 12356), + ('+12345678913', 'user13@example.com', 13.3, 1000013, 12345678913, '123-456-7802', 'Kate', 'Thomas', 'Kate Thomas', 'Sample string 13', 'abcd1246', true, '1234567812345690', 'Category13', 'City13', '1010 Main St, City13', 'Female', '+12345678913', 'sha256_13', '123-45-6801', 'State13', '1010 Main St', 1617281835, 'user13', '2024-05-15T13:51:01.657574Z', 'uuid_13', 12357); diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go index aacaf76854..4327af81c0 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/job_mappings.go @@ -154,6 +154,14 @@ func GetDefaultSyncJobMappings()[]*mgmtv1alpha1.JobMapping { Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH, }, }, + { + Schema: "javascript", + Table: "transformers", + Column: "international_phone", + Transformer: &mgmtv1alpha1.JobMappingTransformer{ + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH, + }, + }, { Schema: "javascript", Table: "transformers", diff --git a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go index 9b4d6e6bcd..572b63a343 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/javascript-transformers/tests.go @@ -32,9 +32,9 @@ func GetSyncTests() []*workflow_testdata.IntegrationTest { func getJsGeneratorJobmappings() []*mgmtv1alpha1.JobMapping { colTransformerMap := map[string]*mgmtv1alpha1.JobMappingTransformer{ - "e164_phone_number": getJavascriptTransformerConfig("return neosync.generateE164PhoneNumber({ min: 9, max: 15});"), + "e164_phone_number": getJavascriptTransformerConfig("return neosync.generateInternationalPhoneNumber({ min: 9, max: 15});"), "email": getJavascriptTransformerConfig("return neosync.generateEmail({ maxLength: 255});"), - "str": getJavascriptTransformerConfig("return neosync.generateString({ min: 1, max: 50});"), + "str": getJavascriptTransformerConfig("return neosync.generateRandomString({ min: 1, max: 50});"), "measurement": getJavascriptTransformerConfig("return neosync.generateFloat64({ min: 3.14, max: 300.10});"), "int64": getJavascriptTransformerConfig("return neosync.generateInt64({ min: 1, max: 50});"), "int64_phone_number": getJavascriptTransformerConfig("return neosync.generateInt64PhoneNumber({});"), @@ -49,6 +49,7 @@ func getJsGeneratorJobmappings() []*mgmtv1alpha1.JobMapping { "city": getJavascriptTransformerConfig("return neosync.generateCity({ maxLength: 100 });"), "full_address": getJavascriptTransformerConfig("return neosync.generateFullAddress({ maxLength: 100 });"), "gender": getJavascriptTransformerConfig("return neosync.generateGender({});"), + "international_phone": getJavascriptTransformerConfig("return neosync.generateInternationalPhoneNumber({ min: 9, max: 14});"), "sha256": getJavascriptTransformerConfig("return neosync.generateSHA256Hash({});"), "ssn": getJavascriptTransformerConfig("return neosync.generateSSN({});"), "state": getJavascriptTransformerConfig("return neosync.generateState({});"), @@ -84,7 +85,7 @@ func getJsTransformerJobmappings() []*mgmtv1alpha1.JobMapping { "measurement": getJavascriptTransformerConfig("return neosync.transformFloat64(value, { randomizationRangeMin: 3.14, randomizationRangeMax: 300.10});"), "int64": getJavascriptTransformerConfig("return neosync.transformInt64(value, { randomizationRangeMin: 1, randomizationRangeMax: 300});"), "int64_phone_number": getJavascriptTransformerConfig("return neosync.transformInt64PhoneNumber(value, { preserveLength: true});"), - "string_phone_number": getJavascriptTransformerConfig("return neosync.transformPhoneNumber(value, { preserveLength: true, maxLength: 200});"), + "string_phone_number": getJavascriptTransformerConfig("return neosync.transformStringPhoneNumber(value, { preserveLength: true, maxLength: 200});"), "first_name": getJavascriptTransformerConfig("return neosync.transformFirstName(value, { preserveLength: true, maxLength: 25});"), "last_name": getJavascriptTransformerConfig("return neosync.transformLastName(value, { preserveLength: true, maxLength: 25});"), "full_name": getJavascriptTransformerConfig("return neosync.transformFullName(value, { preserveLength: true, maxLength: 25});"), From e08437b6c9d8c672840d7125044ffdee35f93d54 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 11:17:50 -0700 Subject: [PATCH 10/32] make transformer config args optional in proto --- .../go/protos/mgmt/v1alpha1/transformer.pb.go | 1103 +++++++++-------- .../protos/mgmt/v1alpha1/transformer.proto | 66 +- docs/protos/data/proto_docs.json | 234 ++-- .../client/mgmt/v1alpha1/transformer_pb.ts | 196 +-- .../neosync_transformer_generator.go | 24 +- .../transformers/transformer_initializer.go | 50 +- .../transformer_initializer_test.go | 106 +- 7 files changed, 951 insertions(+), 828 deletions(-) diff --git a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go index 367ad7cb17..c2cc780d93 100644 --- a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go +++ b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go @@ -2426,9 +2426,9 @@ type TransformEmail struct { unknownFields protoimpl.UnknownFields // Whether or not to preserve the original domain, barring what has been specified in the excluded_domains property. - PreserveDomain bool `protobuf:"varint,1,opt,name=preserve_domain,json=preserveDomain,proto3" json:"preserve_domain,omitempty"` + PreserveDomain *bool `protobuf:"varint,1,opt,name=preserve_domain,json=preserveDomain,proto3,oneof" json:"preserve_domain,omitempty"` // Whether or not to preserve the original length of the email. This causes the transformed email to retain the original length. - PreserveLength bool `protobuf:"varint,2,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,2,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` // A lsit of email domains that should be excluded. This changes based on the preserve_domain flag. See the docs for more details. ExcludedDomains []string `protobuf:"bytes,3,rep,name=excluded_domains,json=excludedDomains,proto3" json:"excluded_domains,omitempty"` // Optionally specify the type of email to generate. The types specified determine the contents on the left side of the @. @@ -2468,15 +2468,15 @@ func (*TransformEmail) Descriptor() ([]byte, []int) { } func (x *TransformEmail) GetPreserveDomain() bool { - if x != nil { - return x.PreserveDomain + if x != nil && x.PreserveDomain != nil { + return *x.PreserveDomain } return false } func (x *TransformEmail) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -2543,7 +2543,7 @@ type GenerateCardNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ValidLuhn bool `protobuf:"varint,1,opt,name=valid_luhn,json=validLuhn,proto3" json:"valid_luhn,omitempty"` + ValidLuhn *bool `protobuf:"varint,1,opt,name=valid_luhn,json=validLuhn,proto3,oneof" json:"valid_luhn,omitempty"` } func (x *GenerateCardNumber) Reset() { @@ -2577,8 +2577,8 @@ func (*GenerateCardNumber) Descriptor() ([]byte, []int) { } func (x *GenerateCardNumber) GetValidLuhn() bool { - if x != nil { - return x.ValidLuhn + if x != nil && x.ValidLuhn != nil { + return *x.ValidLuhn } return false } @@ -2660,8 +2660,8 @@ type GenerateE164PhoneNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Min int64 `protobuf:"varint,1,opt,name=min,proto3" json:"min,omitempty"` - Max int64 `protobuf:"varint,2,opt,name=max,proto3" json:"max,omitempty"` + Min *int64 `protobuf:"varint,1,opt,name=min,proto3,oneof" json:"min,omitempty"` + Max *int64 `protobuf:"varint,2,opt,name=max,proto3,oneof" json:"max,omitempty"` } func (x *GenerateE164PhoneNumber) Reset() { @@ -2695,15 +2695,15 @@ func (*GenerateE164PhoneNumber) Descriptor() ([]byte, []int) { } func (x *GenerateE164PhoneNumber) GetMin() int64 { - if x != nil { - return x.Min + if x != nil && x.Min != nil { + return *x.Min } return 0 } func (x *GenerateE164PhoneNumber) GetMax() int64 { - if x != nil { - return x.Max + if x != nil && x.Max != nil { + return *x.Max } return 0 } @@ -2749,10 +2749,10 @@ type GenerateFloat64 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - RandomizeSign bool `protobuf:"varint,1,opt,name=randomize_sign,json=randomizeSign,proto3" json:"randomize_sign,omitempty"` - Min float64 `protobuf:"fixed64,2,opt,name=min,proto3" json:"min,omitempty"` - Max float64 `protobuf:"fixed64,3,opt,name=max,proto3" json:"max,omitempty"` - Precision int64 `protobuf:"varint,4,opt,name=precision,proto3" json:"precision,omitempty"` + RandomizeSign *bool `protobuf:"varint,1,opt,name=randomize_sign,json=randomizeSign,proto3,oneof" json:"randomize_sign,omitempty"` + Min *float64 `protobuf:"fixed64,2,opt,name=min,proto3,oneof" json:"min,omitempty"` + Max *float64 `protobuf:"fixed64,3,opt,name=max,proto3,oneof" json:"max,omitempty"` + Precision *int64 `protobuf:"varint,4,opt,name=precision,proto3,oneof" json:"precision,omitempty"` } func (x *GenerateFloat64) Reset() { @@ -2786,29 +2786,29 @@ func (*GenerateFloat64) Descriptor() ([]byte, []int) { } func (x *GenerateFloat64) GetRandomizeSign() bool { - if x != nil { - return x.RandomizeSign + if x != nil && x.RandomizeSign != nil { + return *x.RandomizeSign } return false } func (x *GenerateFloat64) GetMin() float64 { - if x != nil { - return x.Min + if x != nil && x.Min != nil { + return *x.Min } return 0 } func (x *GenerateFloat64) GetMax() float64 { - if x != nil { - return x.Max + if x != nil && x.Max != nil { + return *x.Max } return 0 } func (x *GenerateFloat64) GetPrecision() int64 { - if x != nil { - return x.Precision + if x != nil && x.Precision != nil { + return *x.Precision } return 0 } @@ -2890,7 +2890,7 @@ type GenerateGender struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Abbreviate bool `protobuf:"varint,1,opt,name=abbreviate,proto3" json:"abbreviate,omitempty"` + Abbreviate *bool `protobuf:"varint,1,opt,name=abbreviate,proto3,oneof" json:"abbreviate,omitempty"` } func (x *GenerateGender) Reset() { @@ -2924,8 +2924,8 @@ func (*GenerateGender) Descriptor() ([]byte, []int) { } func (x *GenerateGender) GetAbbreviate() bool { - if x != nil { - return x.Abbreviate + if x != nil && x.Abbreviate != nil { + return *x.Abbreviate } return false } @@ -2971,9 +2971,9 @@ type GenerateInt64 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - RandomizeSign bool `protobuf:"varint,1,opt,name=randomize_sign,json=randomizeSign,proto3" json:"randomize_sign,omitempty"` - Min int64 `protobuf:"varint,2,opt,name=min,proto3" json:"min,omitempty"` - Max int64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` + RandomizeSign *bool `protobuf:"varint,1,opt,name=randomize_sign,json=randomizeSign,proto3,oneof" json:"randomize_sign,omitempty"` + Min *int64 `protobuf:"varint,2,opt,name=min,proto3,oneof" json:"min,omitempty"` + Max *int64 `protobuf:"varint,3,opt,name=max,proto3,oneof" json:"max,omitempty"` } func (x *GenerateInt64) Reset() { @@ -3007,22 +3007,22 @@ func (*GenerateInt64) Descriptor() ([]byte, []int) { } func (x *GenerateInt64) GetRandomizeSign() bool { - if x != nil { - return x.RandomizeSign + if x != nil && x.RandomizeSign != nil { + return *x.RandomizeSign } return false } func (x *GenerateInt64) GetMin() int64 { - if x != nil { - return x.Min + if x != nil && x.Min != nil { + return *x.Min } return 0 } func (x *GenerateInt64) GetMax() int64 { - if x != nil { - return x.Max + if x != nil && x.Max != nil { + return *x.Max } return 0 } @@ -3141,7 +3141,7 @@ type GenerateState struct { unknownFields protoimpl.UnknownFields // An option to return the full state name of the randomly selected state or return the default of a 2-letter state code. - GenerateFullName bool `protobuf:"varint,1,opt,name=generate_full_name,json=generateFullName,proto3" json:"generate_full_name,omitempty"` + GenerateFullName *bool `protobuf:"varint,1,opt,name=generate_full_name,json=generateFullName,proto3,oneof" json:"generate_full_name,omitempty"` } func (x *GenerateState) Reset() { @@ -3175,8 +3175,8 @@ func (*GenerateState) Descriptor() ([]byte, []int) { } func (x *GenerateState) GetGenerateFullName() bool { - if x != nil { - return x.GenerateFullName + if x != nil && x.GenerateFullName != nil { + return *x.GenerateFullName } return false } @@ -3222,8 +3222,8 @@ type GenerateStringPhoneNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Min int64 `protobuf:"varint,2,opt,name=min,proto3" json:"min,omitempty"` - Max int64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` + Min *int64 `protobuf:"varint,2,opt,name=min,proto3,oneof" json:"min,omitempty"` + Max *int64 `protobuf:"varint,3,opt,name=max,proto3,oneof" json:"max,omitempty"` } func (x *GenerateStringPhoneNumber) Reset() { @@ -3257,15 +3257,15 @@ func (*GenerateStringPhoneNumber) Descriptor() ([]byte, []int) { } func (x *GenerateStringPhoneNumber) GetMin() int64 { - if x != nil { - return x.Min + if x != nil && x.Min != nil { + return *x.Min } return 0 } func (x *GenerateStringPhoneNumber) GetMax() int64 { - if x != nil { - return x.Max + if x != nil && x.Max != nil { + return *x.Max } return 0 } @@ -3275,8 +3275,8 @@ type GenerateString struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Min int64 `protobuf:"varint,1,opt,name=min,proto3" json:"min,omitempty"` - Max int64 `protobuf:"varint,2,opt,name=max,proto3" json:"max,omitempty"` + Min *int64 `protobuf:"varint,1,opt,name=min,proto3,oneof" json:"min,omitempty"` + Max *int64 `protobuf:"varint,2,opt,name=max,proto3,oneof" json:"max,omitempty"` } func (x *GenerateString) Reset() { @@ -3310,15 +3310,15 @@ func (*GenerateString) Descriptor() ([]byte, []int) { } func (x *GenerateString) GetMin() int64 { - if x != nil { - return x.Min + if x != nil && x.Min != nil { + return *x.Min } return 0 } func (x *GenerateString) GetMax() int64 { - if x != nil { - return x.Max + if x != nil && x.Max != nil { + return *x.Max } return 0 } @@ -3436,7 +3436,7 @@ type GenerateUuid struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - IncludeHyphens bool `protobuf:"varint,1,opt,name=include_hyphens,json=includeHyphens,proto3" json:"include_hyphens,omitempty"` + IncludeHyphens *bool `protobuf:"varint,1,opt,name=include_hyphens,json=includeHyphens,proto3,oneof" json:"include_hyphens,omitempty"` } func (x *GenerateUuid) Reset() { @@ -3470,8 +3470,8 @@ func (*GenerateUuid) Descriptor() ([]byte, []int) { } func (x *GenerateUuid) GetIncludeHyphens() bool { - if x != nil { - return x.IncludeHyphens + if x != nil && x.IncludeHyphens != nil { + return *x.IncludeHyphens } return false } @@ -3517,7 +3517,7 @@ type TransformE164PhoneNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformE164PhoneNumber) Reset() { @@ -3551,8 +3551,8 @@ func (*TransformE164PhoneNumber) Descriptor() ([]byte, []int) { } func (x *TransformE164PhoneNumber) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3562,7 +3562,7 @@ type TransformFirstName struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformFirstName) Reset() { @@ -3596,8 +3596,8 @@ func (*TransformFirstName) Descriptor() ([]byte, []int) { } func (x *TransformFirstName) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3607,8 +3607,8 @@ type TransformFloat64 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - RandomizationRangeMin float64 `protobuf:"fixed64,1,opt,name=randomization_range_min,json=randomizationRangeMin,proto3" json:"randomization_range_min,omitempty"` - RandomizationRangeMax float64 `protobuf:"fixed64,2,opt,name=randomization_range_max,json=randomizationRangeMax,proto3" json:"randomization_range_max,omitempty"` + RandomizationRangeMin *float64 `protobuf:"fixed64,1,opt,name=randomization_range_min,json=randomizationRangeMin,proto3,oneof" json:"randomization_range_min,omitempty"` + RandomizationRangeMax *float64 `protobuf:"fixed64,2,opt,name=randomization_range_max,json=randomizationRangeMax,proto3,oneof" json:"randomization_range_max,omitempty"` } func (x *TransformFloat64) Reset() { @@ -3642,15 +3642,15 @@ func (*TransformFloat64) Descriptor() ([]byte, []int) { } func (x *TransformFloat64) GetRandomizationRangeMin() float64 { - if x != nil { - return x.RandomizationRangeMin + if x != nil && x.RandomizationRangeMin != nil { + return *x.RandomizationRangeMin } return 0 } func (x *TransformFloat64) GetRandomizationRangeMax() float64 { - if x != nil { - return x.RandomizationRangeMax + if x != nil && x.RandomizationRangeMax != nil { + return *x.RandomizationRangeMax } return 0 } @@ -3660,7 +3660,7 @@ type TransformFullName struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformFullName) Reset() { @@ -3694,8 +3694,8 @@ func (*TransformFullName) Descriptor() ([]byte, []int) { } func (x *TransformFullName) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3705,7 +3705,7 @@ type TransformInt64PhoneNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformInt64PhoneNumber) Reset() { @@ -3739,8 +3739,8 @@ func (*TransformInt64PhoneNumber) Descriptor() ([]byte, []int) { } func (x *TransformInt64PhoneNumber) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3750,8 +3750,8 @@ type TransformInt64 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - RandomizationRangeMin int64 `protobuf:"varint,1,opt,name=randomization_range_min,json=randomizationRangeMin,proto3" json:"randomization_range_min,omitempty"` - RandomizationRangeMax int64 `protobuf:"varint,2,opt,name=randomization_range_max,json=randomizationRangeMax,proto3" json:"randomization_range_max,omitempty"` + RandomizationRangeMin *int64 `protobuf:"varint,1,opt,name=randomization_range_min,json=randomizationRangeMin,proto3,oneof" json:"randomization_range_min,omitempty"` + RandomizationRangeMax *int64 `protobuf:"varint,2,opt,name=randomization_range_max,json=randomizationRangeMax,proto3,oneof" json:"randomization_range_max,omitempty"` } func (x *TransformInt64) Reset() { @@ -3785,15 +3785,15 @@ func (*TransformInt64) Descriptor() ([]byte, []int) { } func (x *TransformInt64) GetRandomizationRangeMin() int64 { - if x != nil { - return x.RandomizationRangeMin + if x != nil && x.RandomizationRangeMin != nil { + return *x.RandomizationRangeMin } return 0 } func (x *TransformInt64) GetRandomizationRangeMax() int64 { - if x != nil { - return x.RandomizationRangeMax + if x != nil && x.RandomizationRangeMax != nil { + return *x.RandomizationRangeMax } return 0 } @@ -3803,7 +3803,7 @@ type TransformLastName struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformLastName) Reset() { @@ -3837,8 +3837,8 @@ func (*TransformLastName) Descriptor() ([]byte, []int) { } func (x *TransformLastName) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3848,7 +3848,7 @@ type TransformPhoneNumber struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformPhoneNumber) Reset() { @@ -3882,8 +3882,8 @@ func (*TransformPhoneNumber) Descriptor() ([]byte, []int) { } func (x *TransformPhoneNumber) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -3893,7 +3893,7 @@ type TransformString struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PreserveLength bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3" json:"preserve_length,omitempty"` + PreserveLength *bool `protobuf:"varint,1,opt,name=preserve_length,json=preserveLength,proto3,oneof" json:"preserve_length,omitempty"` } func (x *TransformString) Reset() { @@ -3927,8 +3927,8 @@ func (*TransformString) Descriptor() ([]byte, []int) { } func (x *TransformString) GetPreserveLength() bool { - if x != nil { - return x.PreserveLength + if x != nil && x.PreserveLength != nil { + return *x.PreserveLength } return false } @@ -4199,7 +4199,7 @@ type GenerateCategorical struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Categories string `protobuf:"bytes,1,opt,name=categories,proto3" json:"categories,omitempty"` + Categories *string `protobuf:"bytes,1,opt,name=categories,proto3,oneof" json:"categories,omitempty"` } func (x *GenerateCategorical) Reset() { @@ -4233,8 +4233,8 @@ func (*GenerateCategorical) Descriptor() ([]byte, []int) { } func (x *GenerateCategorical) GetCategories() string { - if x != nil { - return x.Categories + if x != nil && x.Categories != nil { + return *x.Categories } return "" } @@ -4435,7 +4435,7 @@ type GenerateCountry struct { unknownFields protoimpl.UnknownFields // An option to return the full country name of the randomly selected country or return the default of a 2-letter country code. - GenerateFullName bool `protobuf:"varint,1,opt,name=generate_full_name,json=generateFullName,proto3" json:"generate_full_name,omitempty"` + GenerateFullName *bool `protobuf:"varint,1,opt,name=generate_full_name,json=generateFullName,proto3,oneof" json:"generate_full_name,omitempty"` } func (x *GenerateCountry) Reset() { @@ -4469,8 +4469,8 @@ func (*GenerateCountry) Descriptor() ([]byte, []int) { } func (x *GenerateCountry) GetGenerateFullName() bool { - if x != nil { - return x.GenerateFullName + if x != nil && x.GenerateFullName != nil { + return *x.GenerateFullName } return false } @@ -4566,7 +4566,7 @@ type PiiAnonymizer_Mask struct { MaskingChar *string `protobuf:"bytes,1,opt,name=masking_char,json=maskingChar,proto3,oneof" json:"masking_char,omitempty"` // How many characters to mask. CharsToMask *int32 `protobuf:"varint,2,opt,name=chars_to_mask,json=charsToMask,proto3,oneof" json:"chars_to_mask,omitempty"` - // Whether to mask the PII from start or end + // Whether to mask the PII from start or enTransformEmaild FromEnd *bool `protobuf:"varint,3,opt,name=from_end,json=fromEnd,proto3,oneof" json:"from_end,omitempty"` } @@ -5189,452 +5189,495 @@ var file_mgmt_v1alpha1_transformer_proto_rawDesc = []byte{ 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x09, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x6d, - 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd5, 0x02, 0x0a, 0x0e, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x70, + 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x87, 0x03, 0x0a, 0x0e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x29, 0x0a, - 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x64, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x0a, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, - 0x52, 0x09, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x58, - 0x0a, 0x14, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x49, 0x6e, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, - 0x01, 0x52, 0x12, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x6d, 0x61, - 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x69, 0x6e, 0x76, 0x61, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x48, 0x01, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x64, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x44, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x0a, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, + 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x48, 0x02, 0x52, 0x09, 0x65, 0x6d, 0x61, 0x69, + 0x6c, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x58, 0x0a, 0x14, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x0e, 0x0a, 0x0c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x42, 0x6f, 0x6f, 0x6c, - 0x22, 0x33, 0x0a, 0x12, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x61, 0x72, 0x64, - 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, - 0x6c, 0x75, 0x68, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x4c, 0x75, 0x68, 0x6e, 0x22, 0x0e, 0x0a, 0x0c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x43, 0x69, 0x74, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x22, 0x3d, 0x0a, 0x17, 0x47, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x45, 0x31, 0x36, 0x34, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, - 0x62, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x7a, 0x0a, 0x0f, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x36, 0x34, 0x12, - 0x25, 0x0a, 0x0e, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x73, 0x69, 0x67, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, - 0x7a, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, - 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, - 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x15, 0x0a, 0x13, 0x47, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, - 0x12, 0x0a, 0x10, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0x30, 0x0a, 0x0e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x47, - 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x62, 0x62, 0x72, 0x65, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x62, 0x62, 0x72, 0x65, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x22, 0x1a, 0x0a, 0x18, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x22, 0x5a, 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, - 0x36, 0x34, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, - 0x73, 0x69, 0x67, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x72, 0x61, 0x6e, 0x64, - 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, - 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0x12, 0x0a, - 0x10, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, - 0x32, 0x35, 0x36, 0x48, 0x61, 0x73, 0x68, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x53, 0x53, 0x4e, 0x22, 0x3d, 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x10, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, - 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x72, 0x65, 0x65, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x3f, - 0x0a, 0x19, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6d, - 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, - 0x34, 0x0a, 0x0e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, - 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0x17, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x12, - 0x0a, 0x10, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, - 0x6d, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x74, - 0x63, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x37, 0x0a, 0x0c, 0x47, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x75, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x68, 0x79, 0x70, 0x68, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x48, 0x79, 0x70, 0x68, - 0x65, 0x6e, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5a, - 0x69, 0x70, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x43, 0x0a, 0x18, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x45, 0x31, 0x36, 0x34, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x3d, 0x0a, 0x12, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, - 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x82, 0x01, 0x0a, 0x10, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x36, 0x34, 0x12, - 0x36, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, - 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x69, 0x6e, 0x12, 0x36, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, + 0x61, 0x69, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x03, 0x52, 0x12, 0x69, 0x6e, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, + 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, + 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, + 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x69, 0x6e, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x0e, 0x0a, 0x0c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x42, 0x6f, + 0x6f, 0x6c, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x61, + 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x22, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x5f, 0x6c, 0x75, 0x68, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x4c, 0x75, 0x68, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, + 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x6c, 0x75, 0x68, 0x6e, 0x22, 0x0e, 0x0a, 0x0c, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x69, 0x74, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x22, 0x57, + 0x0a, 0x17, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x31, 0x36, 0x34, 0x50, 0x68, + 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, + 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, + 0x03, 0x6d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, + 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xbf, 0x01, 0x0a, + 0x0f, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x36, 0x34, + 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x73, 0x69, + 0x67, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0d, 0x72, 0x61, 0x6e, 0x64, + 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, + 0x6d, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, + 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, + 0x48, 0x02, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a, 0x09, 0x70, 0x72, + 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x03, 0x52, + 0x09, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, + 0x0f, 0x5f, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x73, 0x69, 0x67, 0x6e, + 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, + 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x15, + 0x0a, 0x13, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x44, 0x0a, 0x0e, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x47, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0a, 0x61, + 0x62, 0x62, 0x72, 0x65, 0x76, 0x69, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, + 0x00, 0x52, 0x0a, 0x61, 0x62, 0x62, 0x72, 0x65, 0x76, 0x69, 0x61, 0x74, 0x65, 0x88, 0x01, 0x01, + 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x61, 0x62, 0x62, 0x72, 0x65, 0x76, 0x69, 0x61, 0x74, 0x65, 0x22, + 0x1a, 0x0a, 0x18, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x36, 0x34, + 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0x8c, 0x01, 0x0a, 0x0d, + 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x12, 0x2a, 0x0a, + 0x0e, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0d, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, + 0x7a, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, + 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x48, 0x02, 0x52, + 0x03, 0x6d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x72, 0x61, 0x6e, 0x64, + 0x6f, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, + 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x14, + 0x0a, 0x12, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, + 0x48, 0x61, 0x73, 0x68, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x53, 0x53, 0x4e, 0x22, 0x59, 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x12, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x48, 0x00, 0x52, 0x10, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, + 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x17, + 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x65, 0x74, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x59, 0x0a, 0x19, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, + 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, + 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, + 0x61, 0x78, 0x22, 0x4e, 0x0a, 0x0e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x53, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, + 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, + 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, + 0x61, 0x78, 0x22, 0x17, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, + 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x12, 0x0a, 0x10, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x16, 0x0a, 0x14, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x74, 0x63, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x50, 0x0a, 0x0c, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x55, 0x75, 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x5f, 0x68, 0x79, 0x70, 0x68, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x48, 0x79, 0x70, 0x68, 0x65, + 0x6e, 0x73, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x68, 0x79, 0x70, 0x68, 0x65, 0x6e, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x5a, 0x69, 0x70, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x5c, 0x0a, 0x18, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x45, 0x31, 0x36, 0x34, 0x50, 0x68, 0x6f, + 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x56, 0x0a, 0x12, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, + 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x22, 0xc4, 0x01, 0x0a, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x36, 0x34, 0x12, 0x3b, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, - 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x61, 0x78, 0x22, - 0x3c, 0x0a, 0x11, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6c, 0x6c, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x44, 0x0a, - 0x19, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x50, - 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, - 0x67, 0x74, 0x68, 0x22, 0x80, 0x01, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x12, 0x36, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x69, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, - 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x69, 0x6e, 0x12, 0x36, - 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, - 0x6e, 0x67, 0x65, 0x4d, 0x61, 0x78, 0x22, 0x3c, 0x0a, 0x11, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, - 0x6e, 0x67, 0x74, 0x68, 0x22, 0x3f, 0x0a, 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x3a, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, - 0x72, 0x6d, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, + 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, + 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x69, + 0x6e, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x78, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x61, 0x78, 0x88, 0x01, + 0x01, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x1a, 0x0a, + 0x18, 0x5f, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x78, 0x22, 0x55, 0x0a, 0x11, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6c, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2c, + 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, + 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x22, 0x5d, 0x0a, 0x19, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x74, + 0x36, 0x34, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x2c, 0x0a, + 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, + 0xc2, 0x01, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x74, + 0x36, 0x34, 0x12, 0x3b, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x69, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, + 0x3b, 0x0a, 0x17, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, + 0x48, 0x01, 0x52, 0x15, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x42, 0x1a, 0x0a, 0x18, + 0x5f, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, + 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x72, 0x61, 0x6e, + 0x64, 0x6f, 0x6d, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, + 0x5f, 0x6d, 0x61, 0x78, 0x22, 0x55, 0x0a, 0x11, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x58, 0x0a, 0x14, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x88, 0x01, + 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x53, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x72, 0x6d, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x22, 0x0d, 0x0a, 0x0b, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, - 0x22, 0x06, 0x0a, 0x04, 0x4e, 0x75, 0x6c, 0x6c, 0x22, 0x29, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x12, + 0x08, 0x48, 0x00, 0x52, 0x0e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x0d, 0x0a, 0x0b, 0x50, 0x61, + 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x22, 0x06, 0x0a, 0x04, 0x4e, 0x75, 0x6c, + 0x6c, 0x22, 0x29, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x4a, 0x61, + 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x38, 0x0a, 0x1c, + 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, + 0x01, 0x01, 0x52, 0x02, 0x69, 0x64, 0x22, 0x5f, 0x0a, 0x21, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, + 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xba, 0x48, 0x04, 0x72, 0x02, 0x10, + 0x01, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x3a, 0x0a, 0x22, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x22, 0x49, 0x0a, 0x13, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, + 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x0a, 0x63, 0x61, + 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, + 0x52, 0x0a, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x88, 0x01, 0x01, 0x42, + 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0x69, + 0x0a, 0x1a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x61, + 0x63, 0x74, 0x65, 0x72, 0x53, 0x63, 0x72, 0x61, 0x6d, 0x62, 0x6c, 0x65, 0x12, 0x33, 0x0a, 0x13, + 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x5f, 0x72, 0x65, + 0x67, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x11, 0x75, 0x73, 0x65, + 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x52, 0x65, 0x67, 0x65, 0x78, 0x88, 0x01, + 0x01, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x22, 0x28, 0x0a, 0x12, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, - 0x6f, 0x64, 0x65, 0x22, 0x38, 0x0a, 0x1c, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x02, 0x69, 0x64, 0x22, 0x5f, 0x0a, - 0x21, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, - 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x07, 0xba, 0x48, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x3a, - 0x0a, 0x22, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, - 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x35, 0x0a, 0x13, 0x47, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x63, 0x61, - 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x69, 0x65, - 0x73, 0x22, 0x69, 0x0a, 0x1a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x43, 0x68, - 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x53, 0x63, 0x72, 0x61, 0x6d, 0x62, 0x6c, 0x65, 0x12, - 0x33, 0x0a, 0x13, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, - 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x11, - 0x75, 0x73, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x52, 0x65, 0x67, 0x65, - 0x78, 0x88, 0x01, 0x01, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x22, 0x28, 0x0a, 0x12, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x6d, 0x0a, 0x1c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, - 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x11, 0x75, 0x73, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, - 0x52, 0x65, 0x67, 0x65, 0x78, 0x22, 0x35, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x3f, 0x0a, 0x0f, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x2c, 0x0a, 0x12, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x66, 0x75, 0x6c, 0x6c, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0x85, 0x10, - 0x0a, 0x11, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x12, 0x22, 0x0a, 0x1e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, - 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x22, 0x0a, 0x1e, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x50, 0x41, - 0x53, 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, 0x47, 0x48, 0x10, 0x01, 0x12, 0x27, 0x0a, 0x23, 0x54, - 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, - 0x4c, 0x54, 0x10, 0x02, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, - 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x4a, 0x41, 0x56, 0x41, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, - 0x03, 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, - 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, - 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, 0x04, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, 0x41, 0x4e, - 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, 0x05, - 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, - 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x06, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, - 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, - 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x52, 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, - 0x52, 0x10, 0x07, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, - 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, - 0x54, 0x45, 0x5f, 0x43, 0x49, 0x54, 0x59, 0x10, 0x08, 0x12, 0x31, 0x0a, 0x2d, 0x54, 0x52, 0x41, + 0x6f, 0x64, 0x65, 0x22, 0x6d, 0x0a, 0x1c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, + 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x11, 0x75, 0x73, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x52, 0x65, 0x67, + 0x65, 0x78, 0x22, 0x35, 0x0a, 0x1d, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, + 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x5b, 0x0a, 0x0f, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x31, 0x0a, 0x12, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x10, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, + 0x15, 0x0a, 0x13, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x66, 0x75, 0x6c, + 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x2a, 0x85, 0x10, 0x0a, 0x11, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x22, 0x0a, 0x1e, + 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x22, 0x0a, 0x1e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x50, 0x41, 0x53, 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, + 0x47, 0x48, 0x10, 0x01, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, + 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x02, 0x12, 0x2b, 0x0a, + 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x4a, 0x41, + 0x56, 0x41, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x03, 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, + 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, + 0x04, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, + 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x4d, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, 0x05, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x31, 0x36, 0x34, 0x5f, 0x50, 0x48, - 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x09, 0x12, 0x2a, 0x0a, 0x26, + 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x06, 0x12, + 0x2b, 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, + 0x41, 0x52, 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x07, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x49, 0x52, 0x53, - 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x0a, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, - 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, - 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x36, 0x34, 0x10, - 0x0b, 0x12, 0x2c, 0x0a, 0x28, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, + 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x49, 0x54, 0x59, + 0x10, 0x08, 0x12, 0x31, 0x0a, 0x2d, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, + 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, + 0x45, 0x5f, 0x45, 0x31, 0x36, 0x34, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x10, 0x09, 0x12, 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, + 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, + 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x49, 0x52, 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, + 0x0a, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, - 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0c, 0x12, - 0x29, 0x0a, 0x25, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, - 0x55, 0x4c, 0x4c, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x0d, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, + 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x36, 0x34, 0x10, 0x0b, 0x12, 0x2c, 0x0a, 0x28, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x44, 0x45, 0x52, - 0x10, 0x0e, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, - 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, - 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, - 0x4d, 0x42, 0x45, 0x52, 0x10, 0x0f, 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, - 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, - 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x10, 0x12, 0x2c, 0x0a, - 0x28, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x4e, - 0x44, 0x4f, 0x4d, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x11, 0x12, 0x29, 0x0a, 0x25, 0x54, + 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x5f, 0x41, + 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0c, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x52, 0x41, 0x4e, + 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, + 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x5f, 0x4e, 0x41, 0x4d, + 0x45, 0x10, 0x0d, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, + 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, + 0x54, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x44, 0x45, 0x52, 0x10, 0x0e, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x4c, 0x41, 0x53, 0x54, 0x5f, - 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, + 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, + 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x0f, 0x12, + 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x49, + 0x4e, 0x54, 0x36, 0x34, 0x10, 0x10, 0x12, 0x2c, 0x0a, 0x28, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, - 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x48, 0x41, 0x53, 0x48, - 0x10, 0x13, 0x12, 0x23, 0x0a, 0x1f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, - 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, - 0x45, 0x5f, 0x53, 0x53, 0x4e, 0x10, 0x14, 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, + 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x5f, 0x49, 0x4e, 0x54, + 0x36, 0x34, 0x10, 0x11, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, + 0x41, 0x54, 0x45, 0x5f, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x12, 0x12, + 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, + 0x48, 0x41, 0x32, 0x35, 0x36, 0x48, 0x41, 0x53, 0x48, 0x10, 0x13, 0x12, 0x23, 0x0a, 0x1f, 0x54, + 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x53, 0x4e, 0x10, 0x14, + 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x10, 0x15, 0x12, 0x2e, 0x0a, 0x2a, 0x54, 0x52, 0x41, 0x4e, 0x53, + 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, + 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x45, 0x45, 0x54, 0x5f, 0x41, 0x44, + 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x16, 0x12, 0x33, 0x0a, 0x2f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, - 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x10, 0x15, 0x12, 0x2e, - 0x0a, 0x2a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, - 0x52, 0x45, 0x45, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x16, 0x12, 0x33, - 0x0a, 0x2f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, - 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, - 0x52, 0x10, 0x17, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, + 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x48, + 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x17, 0x12, 0x26, 0x0a, 0x22, + 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, + 0x4e, 0x47, 0x10, 0x18, 0x12, 0x2d, 0x0a, 0x29, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, + 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x4e, 0x44, 0x4f, 0x4d, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, + 0x47, 0x10, 0x19, 0x12, 0x2d, 0x0a, 0x29, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, - 0x54, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x18, 0x12, 0x2d, 0x0a, 0x29, 0x54, - 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x4e, 0x44, 0x4f, - 0x4d, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x19, 0x12, 0x2d, 0x0a, 0x29, 0x54, 0x52, + 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x58, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, + 0x10, 0x1a, 0x12, 0x28, 0x0a, 0x24, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, + 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, + 0x45, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x2c, 0x0a, 0x28, + 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x54, 0x43, 0x54, + 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x1c, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x58, 0x54, 0x49, - 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x1a, 0x12, 0x28, 0x0a, 0x24, 0x54, 0x52, 0x41, + 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x1d, + 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, + 0x5a, 0x49, 0x50, 0x43, 0x4f, 0x44, 0x45, 0x10, 0x1e, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x4e, 0x41, 0x4d, - 0x45, 0x10, 0x1b, 0x12, 0x2c, 0x0a, 0x28, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, - 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, - 0x54, 0x45, 0x5f, 0x55, 0x54, 0x43, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, - 0x1c, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, - 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, - 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x1d, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, - 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x5a, 0x49, 0x50, 0x43, 0x4f, 0x44, 0x45, 0x10, 0x1e, - 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, - 0x5f, 0x45, 0x31, 0x36, 0x34, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, - 0x45, 0x52, 0x10, 0x1f, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, - 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x46, 0x49, 0x52, 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, - 0x20, 0x12, 0x28, 0x0a, 0x24, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, - 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, - 0x4d, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x36, 0x34, 0x10, 0x21, 0x12, 0x2a, 0x0a, 0x26, 0x54, - 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x46, 0x55, 0x4c, 0x4c, - 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x22, 0x12, 0x33, 0x0a, 0x2f, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, - 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x5f, 0x50, 0x48, - 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x23, 0x12, 0x26, 0x0a, 0x22, - 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x49, 0x4e, 0x54, - 0x36, 0x34, 0x10, 0x24, 0x12, 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x45, 0x31, 0x36, 0x34, 0x5f, 0x50, + 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x1f, 0x12, 0x2b, 0x0a, + 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x46, 0x49, + 0x52, 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x20, 0x12, 0x28, 0x0a, 0x24, 0x54, 0x52, + 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, + 0x36, 0x34, 0x10, 0x21, 0x12, 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x25, - 0x12, 0x2d, 0x0a, 0x29, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, + 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x22, + 0x12, 0x33, 0x0a, 0x2f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, - 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x26, 0x12, - 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, - 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x27, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, - 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, - 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x28, 0x12, 0x2b, - 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, - 0x54, 0x45, 0x47, 0x4f, 0x52, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x2a, 0x12, 0x33, 0x0a, 0x2f, 0x54, - 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x43, 0x48, 0x41, 0x52, - 0x41, 0x43, 0x54, 0x45, 0x52, 0x5f, 0x53, 0x43, 0x52, 0x41, 0x4d, 0x42, 0x4c, 0x45, 0x10, 0x2b, - 0x12, 0x23, 0x0a, 0x1f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x46, 0x49, - 0x4e, 0x45, 0x44, 0x10, 0x2c, 0x12, 0x2a, 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, - 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, - 0x52, 0x41, 0x54, 0x45, 0x5f, 0x4a, 0x41, 0x56, 0x41, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, - 0x2d, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, + 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x10, 0x23, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, + 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, + 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x24, 0x12, 0x2a, 0x0a, + 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x4c, 0x41, + 0x53, 0x54, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x25, 0x12, 0x2d, 0x0a, 0x29, 0x54, 0x52, 0x41, + 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x50, 0x48, 0x4f, 0x4e, 0x45, 0x5f, + 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x26, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, + 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, + 0x27, 0x12, 0x24, 0x0a, 0x20, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, - 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x2e, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x52, + 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x28, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x52, 0x41, 0x4e, 0x53, + 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, + 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x54, 0x45, 0x47, 0x4f, 0x52, 0x49, 0x43, + 0x41, 0x4c, 0x10, 0x2a, 0x12, 0x33, 0x0a, 0x2f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, + 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, + 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x43, 0x48, 0x41, 0x52, 0x41, 0x43, 0x54, 0x45, 0x52, 0x5f, 0x53, + 0x43, 0x52, 0x41, 0x4d, 0x42, 0x4c, 0x45, 0x10, 0x2b, 0x12, 0x23, 0x0a, 0x1f, 0x54, 0x52, 0x41, + 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x55, 0x53, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x2c, 0x12, 0x2a, + 0x0a, 0x26, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x4a, 0x41, + 0x56, 0x41, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x2d, 0x12, 0x27, 0x0a, 0x23, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x50, 0x49, 0x49, 0x5f, 0x54, - 0x45, 0x58, 0x54, 0x10, 0x2f, 0x2a, 0xc4, 0x02, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, - 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, - 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, - 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, - 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, + 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x52, + 0x59, 0x10, 0x2e, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, + 0x45, 0x52, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, + 0x4f, 0x52, 0x4d, 0x5f, 0x50, 0x49, 0x49, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x10, 0x2f, 0x2a, 0xc4, + 0x02, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x54, 0x52, 0x41, 0x4e, 0x53, - 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x45, 0x41, 0x4e, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x54, 0x52, - 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x36, 0x34, 0x10, 0x04, 0x12, 0x1e, 0x0a, - 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, - 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x05, 0x12, 0x1d, 0x0a, - 0x19, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, - 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x4e, 0x59, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, - 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x07, 0x12, 0x1e, 0x0a, 0x1a, - 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x08, 0x2a, 0x74, 0x0a, 0x10, - 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x22, 0x0a, 0x1e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x4a, 0x4f, - 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, - 0x44, 0x5f, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x10, - 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x4a, - 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, - 0x10, 0x02, 0x2a, 0x7b, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6d, - 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x1f, 0x47, 0x45, 0x4e, 0x45, 0x52, - 0x41, 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, - 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x5f, 0x56, 0x34, 0x10, 0x01, 0x12, 0x20, 0x0a, - 0x1c, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x02, 0x2a, - 0xc3, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, 0x61, 0x69, 0x6c, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x20, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, - 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4a, 0x45, 0x43, 0x54, 0x10, 0x01, 0x12, 0x1d, 0x0a, - 0x19, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, - 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x41, 0x53, 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, 0x47, 0x48, - 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, - 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, - 0x41, 0x54, 0x45, 0x10, 0x04, 0x32, 0xd2, 0x0a, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x74, 0x0a, - 0x15, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x2b, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x12, 0x32, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x83, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x30, - 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8c, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, - 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x65, 0x72, 0x42, 0x79, 0x49, 0x64, 0x12, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, - 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, - 0x72, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x6d, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, + 0x1c, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, + 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, + 0x1f, 0x0a, 0x1b, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, + 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x02, + 0x12, 0x21, 0x0a, 0x1d, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, + 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x45, 0x41, + 0x4e, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, + 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x4c, 0x4f, + 0x41, 0x54, 0x36, 0x34, 0x10, 0x04, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, + 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x05, 0x12, 0x1d, 0x0a, 0x19, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, + 0x4f, 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x41, 0x4e, 0x59, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, + 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, + 0x49, 0x4d, 0x45, 0x10, 0x07, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, + 0x52, 0x4d, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, + 0x55, 0x49, 0x44, 0x10, 0x08, 0x2a, 0x74, 0x0a, 0x10, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, + 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x0a, 0x1e, 0x53, 0x55, 0x50, + 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, + 0x17, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x4a, 0x4f, 0x42, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x55, + 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x10, 0x02, 0x2a, 0x7b, 0x0a, 0x11, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x23, 0x0a, 0x1f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, + 0x49, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, + 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x55, 0x49, + 0x44, 0x5f, 0x56, 0x34, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, + 0x54, 0x45, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x55, + 0x4c, 0x4c, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x02, 0x2a, 0xc3, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x24, 0x0a, 0x20, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, + 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, + 0x4a, 0x45, 0x43, 0x54, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, + 0x55, 0x4c, 0x4c, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x41, + 0x53, 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, 0x47, 0x48, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x49, + 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x5f, 0x41, 0x43, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x41, 0x54, 0x45, 0x10, 0x04, 0x32, 0xd2, + 0x0a, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x74, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x12, + 0x2b, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x72, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, + 0x1c, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x32, 0x2e, + 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, + 0x65, 0x72, 0x42, 0x79, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, - 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x12, 0x32, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, - 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x89, 0x01, 0x0a, 0x1c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, - 0x72, 0x12, 0x32, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, - 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, - 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, - 0x1c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x12, 0x32, 0x2e, - 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, - 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, 0x01, 0x0a, 0x1a, 0x49, 0x73, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x41, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x30, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x49, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, - 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x49, 0x73, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, - 0x01, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, - 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x30, 0x2e, - 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x15, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x2e, - 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, - 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xcc, 0x01, 0x0a, 0x11, 0x63, - 0x6f, 0x6d, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x42, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x50, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6e, 0x75, 0x63, 0x6c, 0x65, 0x75, 0x73, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6e, 0x65, - 0x6f, 0x73, 0x79, 0x6e, 0x63, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x67, 0x65, - 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x6d, 0x67, 0x6d, 0x74, - 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, 0x6d, 0x67, 0x6d, 0x74, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0d, 0x4d, - 0x67, 0x6d, 0x74, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xca, 0x02, 0x0d, 0x4d, - 0x67, 0x6d, 0x74, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 0x02, 0x19, 0x4d, - 0x67, 0x6d, 0x74, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x4d, 0x67, 0x6d, 0x74, 0x3a, - 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, + 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8c, + 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, + 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x49, 0x64, + 0x12, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, + 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x42, + 0x79, 0x49, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, + 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, + 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x12, 0x32, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, + 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, 0x1c, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x12, 0x32, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x89, 0x01, 0x0a, 0x1c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x12, 0x32, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, + 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x83, 0x01, 0x0a, 0x1a, 0x49, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, + 0x12, 0x30, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x49, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x4e, 0x61, + 0x6d, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x49, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, 0x01, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x30, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, + 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x43, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, + 0x15, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, + 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, + 0x73, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, + 0x52, 0x65, 0x67, 0x65, 0x78, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x42, 0xcc, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x42, 0x10, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x50, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x63, 0x6c, 0x65, 0x75, + 0x73, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6e, 0x65, 0x6f, 0x73, 0x79, 0x6e, 0x63, 0x2f, 0x62, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x3b, 0x6d, 0x67, 0x6d, 0x74, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xa2, + 0x02, 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0d, 0x4d, 0x67, 0x6d, 0x74, 0x2e, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0xca, 0x02, 0x0d, 0x4d, 0x67, 0x6d, 0x74, 0x5c, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 0x02, 0x19, 0x4d, 0x67, 0x6d, 0x74, 0x5c, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0xea, 0x02, 0x0e, 0x4d, 0x67, 0x6d, 0x74, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -5895,7 +5938,27 @@ func file_mgmt_v1alpha1_transformer_proto_init() { } file_mgmt_v1alpha1_transformer_proto_msgTypes[22].OneofWrappers = []any{} file_mgmt_v1alpha1_transformer_proto_msgTypes[23].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[25].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[28].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[30].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[33].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[35].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[39].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[41].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[42].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[46].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[48].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[49].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[50].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[51].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[52].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[53].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[54].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[55].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[56].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[63].OneofWrappers = []any{} file_mgmt_v1alpha1_transformer_proto_msgTypes[64].OneofWrappers = []any{} + file_mgmt_v1alpha1_transformer_proto_msgTypes[68].OneofWrappers = []any{} file_mgmt_v1alpha1_transformer_proto_msgTypes[69].OneofWrappers = []any{} file_mgmt_v1alpha1_transformer_proto_msgTypes[71].OneofWrappers = []any{} file_mgmt_v1alpha1_transformer_proto_msgTypes[72].OneofWrappers = []any{} diff --git a/backend/protos/mgmt/v1alpha1/transformer.proto b/backend/protos/mgmt/v1alpha1/transformer.proto index 2d63dcd002..719a7564bb 100644 --- a/backend/protos/mgmt/v1alpha1/transformer.proto +++ b/backend/protos/mgmt/v1alpha1/transformer.proto @@ -267,7 +267,7 @@ message PiiAnonymizer { optional string masking_char = 1; // How many characters to mask. optional int32 chars_to_mask = 2; - // Whether to mask the PII from start or end + // Whether to mask the PII from start or enTransformEmaild optional bool from_end = 3; } message Hash { @@ -317,9 +317,9 @@ enum InvalidEmailAction { message TransformEmail { // Whether or not to preserve the original domain, barring what has been specified in the excluded_domains property. - bool preserve_domain = 1; + optional bool preserve_domain = 1; // Whether or not to preserve the original length of the email. This causes the transformed email to retain the original length. - bool preserve_length = 2; + optional bool preserve_length = 2; // A lsit of email domains that should be excluded. This changes based on the preserve_domain flag. See the docs for more details. repeated string excluded_domains = 3; // Optionally specify the type of email to generate. The types specified determine the contents on the left side of the @. @@ -331,7 +331,7 @@ message TransformEmail { message GenerateBool {} message GenerateCardNumber { - bool valid_luhn = 1; + optional bool valid_luhn = 1; } message GenerateCity {} @@ -339,17 +339,17 @@ message GenerateCity {} message GenerateDefault {} message GenerateE164PhoneNumber { - int64 min = 1; - int64 max = 2; + optional int64 min = 1; + optional int64 max = 2; } message GenerateFirstName {} message GenerateFloat64 { - bool randomize_sign = 1; - double min = 2; - double max = 3; - int64 precision = 4; + optional bool randomize_sign = 1; + optional double min = 2; + optional double max = 3; + optional int64 precision = 4; } message GenerateFullAddress {} @@ -357,15 +357,15 @@ message GenerateFullAddress {} message GenerateFullName {} message GenerateGender { - bool abbreviate = 1; + optional bool abbreviate = 1; } message GenerateInt64PhoneNumber {} message GenerateInt64 { - bool randomize_sign = 1; - int64 min = 2; - int64 max = 3; + optional bool randomize_sign = 1; + optional int64 min = 2; + optional int64 max = 3; } message GenerateLastName {} @@ -375,19 +375,19 @@ message GenerateSSN {} message GenerateState { // An option to return the full state name of the randomly selected state or return the default of a 2-letter state code. - bool generate_full_name = 1; + optional bool generate_full_name = 1; } message GenerateStreetAddress {} message GenerateStringPhoneNumber { - int64 min = 2; - int64 max = 3; + optional int64 min = 2; + optional int64 max = 3; } message GenerateString { - int64 min = 1; - int64 max = 2; + optional int64 min = 1; + optional int64 max = 2; } message GenerateUnixTimestamp {} @@ -397,47 +397,47 @@ message GenerateUsername {} message GenerateUtcTimestamp {} message GenerateUuid { - bool include_hyphens = 1; + optional bool include_hyphens = 1; } message GenerateZipcode {} message TransformE164PhoneNumber { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformFirstName { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformFloat64 { - double randomization_range_min = 1; - double randomization_range_max = 2; + optional double randomization_range_min = 1; + optional double randomization_range_max = 2; } message TransformFullName { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformInt64PhoneNumber { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformInt64 { - int64 randomization_range_min = 1; - int64 randomization_range_max = 2; + optional int64 randomization_range_min = 1; + optional int64 randomization_range_max = 2; } message TransformLastName { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformPhoneNumber { - bool preserve_length = 1; + optional bool preserve_length = 1; } message TransformString { - bool preserve_length = 1; + optional bool preserve_length = 1; } message Passthrough {} @@ -463,7 +463,7 @@ message ValidateUserJavascriptCodeResponse { } message GenerateCategorical { - string categories = 1; + optional string categories = 1; } message TransformCharacterScramble { @@ -487,7 +487,7 @@ message ValidateUserRegexCodeResponse { message GenerateCountry { // An option to return the full country name of the randomly selected country or return the default of a 2-letter country code. - bool generate_full_name = 1; + optional bool generate_full_name = 1; } service TransformersService { diff --git a/docs/protos/data/proto_docs.json b/docs/protos/data/proto_docs.json index f8c81a8d91..062a3b1a81 100644 --- a/docs/protos/data/proto_docs.json +++ b/docs/protos/data/proto_docs.json @@ -576,19 +576,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "valid_luhn", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_valid_luhn", "defaultValue": "" } ] @@ -600,19 +600,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "categories", "description": "", - "label": "", + "label": "optional", "type": "string", "longType": "string", "fullType": "string", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_categories", "defaultValue": "" } ] @@ -635,19 +635,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "generate_full_name", "description": "An option to return the full country name of the randomly selected country or return the default of a 2-letter country code.", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_generate_full_name", "defaultValue": "" } ] @@ -670,31 +670,31 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "min", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_min", "defaultValue": "" }, { "name": "max", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_max", "defaultValue": "" } ] @@ -741,55 +741,55 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "randomize_sign", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomize_sign", "defaultValue": "" }, { "name": "min", "description": "", - "label": "", + "label": "optional", "type": "double", "longType": "double", "fullType": "double", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_min", "defaultValue": "" }, { "name": "max", "description": "", - "label": "", + "label": "optional", "type": "double", "longType": "double", "fullType": "double", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_max", "defaultValue": "" }, { "name": "precision", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_precision", "defaultValue": "" } ] @@ -823,19 +823,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "abbreviate", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_abbreviate", "defaultValue": "" } ] @@ -847,43 +847,43 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "randomize_sign", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomize_sign", "defaultValue": "" }, { "name": "min", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_min", "defaultValue": "" }, { "name": "max", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_max", "defaultValue": "" } ] @@ -963,19 +963,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "generate_full_name", "description": "An option to return the full state name of the randomly selected state or return the default of a 2-letter state code.", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_generate_full_name", "defaultValue": "" } ] @@ -998,31 +998,31 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "min", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_min", "defaultValue": "" }, { "name": "max", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_max", "defaultValue": "" } ] @@ -1034,31 +1034,31 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "min", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_min", "defaultValue": "" }, { "name": "max", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_max", "defaultValue": "" } ] @@ -1103,19 +1103,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "include_hyphens", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_include_hyphens", "defaultValue": "" } ] @@ -1512,7 +1512,7 @@ }, { "name": "from_end", - "description": "Whether to mask the PII from start or end", + "description": "Whether to mask the PII from start or enTransformEmaild", "label": "optional", "type": "bool", "longType": "bool", @@ -1722,19 +1722,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -1752,25 +1752,25 @@ { "name": "preserve_domain", "description": "Whether or not to preserve the original domain, barring what has been specified in the excluded_domains property.", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_domain", "defaultValue": "" }, { "name": "preserve_length", "description": "Whether or not to preserve the original length of the email. This causes the transformed email to retain the original length.", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" }, { @@ -1818,19 +1818,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -1842,31 +1842,31 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "randomization_range_min", "description": "", - "label": "", + "label": "optional", "type": "double", "longType": "double", "fullType": "double", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomization_range_min", "defaultValue": "" }, { "name": "randomization_range_max", "description": "", - "label": "", + "label": "optional", "type": "double", "longType": "double", "fullType": "double", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomization_range_max", "defaultValue": "" } ] @@ -1878,19 +1878,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -1902,31 +1902,31 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "randomization_range_min", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomization_range_min", "defaultValue": "" }, { "name": "randomization_range_max", "description": "", - "label": "", + "label": "optional", "type": "int64", "longType": "int64", "fullType": "int64", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_randomization_range_max", "defaultValue": "" } ] @@ -1938,19 +1938,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -1986,19 +1986,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -2010,19 +2010,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] @@ -2082,19 +2082,19 @@ "description": "", "hasExtensions": false, "hasFields": true, - "hasOneofs": false, + "hasOneofs": true, "extensions": [], "fields": [ { "name": "preserve_length", "description": "", - "label": "", + "label": "optional", "type": "bool", "longType": "bool", "fullType": "bool", "ismap": false, - "isoneof": false, - "oneofdecl": "", + "isoneof": true, + "oneofdecl": "_preserve_length", "defaultValue": "" } ] diff --git a/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts b/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts index f81f064ac2..9a320fe6ad 100644 --- a/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts +++ b/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts @@ -4,7 +4,7 @@ // @ts-nocheck import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; -import { Message, proto3, protoInt64, Timestamp } from "@bufbuild/protobuf"; +import { Message, proto3, Timestamp } from "@bufbuild/protobuf"; /** * @generated from enum mgmt.v1alpha1.TransformerSource @@ -1881,7 +1881,7 @@ export class PiiAnonymizer_Mask extends Message { charsToMask?: number; /** - * Whether to mask the PII from start or end + * Whether to mask the PII from start or enTransformEmaild * * @generated from field: optional bool from_end = 3; */ @@ -2042,16 +2042,16 @@ export class TransformEmail extends Message { /** * Whether or not to preserve the original domain, barring what has been specified in the excluded_domains property. * - * @generated from field: bool preserve_domain = 1; + * @generated from field: optional bool preserve_domain = 1; */ - preserveDomain = false; + preserveDomain?: boolean; /** * Whether or not to preserve the original length of the email. This causes the transformed email to retain the original length. * - * @generated from field: bool preserve_length = 2; + * @generated from field: optional bool preserve_length = 2; */ - preserveLength = false; + preserveLength?: boolean; /** * A lsit of email domains that should be excluded. This changes based on the preserve_domain flag. See the docs for more details. @@ -2082,8 +2082,8 @@ export class TransformEmail extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformEmail"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_domain", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 2, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_domain", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 2, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, { no: 3, name: "excluded_domains", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, { no: 4, name: "email_type", kind: "enum", T: proto3.getEnumType(GenerateEmailType), opt: true }, { no: 5, name: "invalid_email_action", kind: "enum", T: proto3.getEnumType(InvalidEmailAction), opt: true }, @@ -2142,9 +2142,9 @@ export class GenerateBool extends Message { */ export class GenerateCardNumber extends Message { /** - * @generated from field: bool valid_luhn = 1; + * @generated from field: optional bool valid_luhn = 1; */ - validLuhn = false; + validLuhn?: boolean; constructor(data?: PartialMessage) { super(); @@ -2154,7 +2154,7 @@ export class GenerateCardNumber extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateCardNumber"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "valid_luhn", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "valid_luhn", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateCardNumber { @@ -2241,14 +2241,14 @@ export class GenerateDefault extends Message { */ export class GenerateE164PhoneNumber extends Message { /** - * @generated from field: int64 min = 1; + * @generated from field: optional int64 min = 1; */ - min = protoInt64.zero; + min?: bigint; /** - * @generated from field: int64 max = 2; + * @generated from field: optional int64 max = 2; */ - max = protoInt64.zero; + max?: bigint; constructor(data?: PartialMessage) { super(); @@ -2258,8 +2258,8 @@ export class GenerateE164PhoneNumber extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateE164PhoneNumber"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 2, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 1, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 2, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateE164PhoneNumber { @@ -2315,24 +2315,24 @@ export class GenerateFirstName extends Message { */ export class GenerateFloat64 extends Message { /** - * @generated from field: bool randomize_sign = 1; + * @generated from field: optional bool randomize_sign = 1; */ - randomizeSign = false; + randomizeSign?: boolean; /** - * @generated from field: double min = 2; + * @generated from field: optional double min = 2; */ - min = 0; + min?: number; /** - * @generated from field: double max = 3; + * @generated from field: optional double max = 3; */ - max = 0; + max?: number; /** - * @generated from field: int64 precision = 4; + * @generated from field: optional int64 precision = 4; */ - precision = protoInt64.zero; + precision?: bigint; constructor(data?: PartialMessage) { super(); @@ -2342,10 +2342,10 @@ export class GenerateFloat64 extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateFloat64"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "randomize_sign", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 2, name: "min", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, - { no: 3, name: "max", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, - { no: 4, name: "precision", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 1, name: "randomize_sign", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 2, name: "min", kind: "scalar", T: 1 /* ScalarType.DOUBLE */, opt: true }, + { no: 3, name: "max", kind: "scalar", T: 1 /* ScalarType.DOUBLE */, opt: true }, + { no: 4, name: "precision", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateFloat64 { @@ -2432,9 +2432,9 @@ export class GenerateFullName extends Message { */ export class GenerateGender extends Message { /** - * @generated from field: bool abbreviate = 1; + * @generated from field: optional bool abbreviate = 1; */ - abbreviate = false; + abbreviate?: boolean; constructor(data?: PartialMessage) { super(); @@ -2444,7 +2444,7 @@ export class GenerateGender extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateGender"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "abbreviate", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "abbreviate", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateGender { @@ -2500,19 +2500,19 @@ export class GenerateInt64PhoneNumber extends Message */ export class GenerateInt64 extends Message { /** - * @generated from field: bool randomize_sign = 1; + * @generated from field: optional bool randomize_sign = 1; */ - randomizeSign = false; + randomizeSign?: boolean; /** - * @generated from field: int64 min = 2; + * @generated from field: optional int64 min = 2; */ - min = protoInt64.zero; + min?: bigint; /** - * @generated from field: int64 max = 3; + * @generated from field: optional int64 max = 3; */ - max = protoInt64.zero; + max?: bigint; constructor(data?: PartialMessage) { super(); @@ -2522,9 +2522,9 @@ export class GenerateInt64 extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateInt64"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "randomize_sign", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 2, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 3, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 1, name: "randomize_sign", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 2, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 3, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateInt64 { @@ -2644,9 +2644,9 @@ export class GenerateState extends Message { /** * An option to return the full state name of the randomly selected state or return the default of a 2-letter state code. * - * @generated from field: bool generate_full_name = 1; + * @generated from field: optional bool generate_full_name = 1; */ - generateFullName = false; + generateFullName?: boolean; constructor(data?: PartialMessage) { super(); @@ -2656,7 +2656,7 @@ export class GenerateState extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateState"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "generate_full_name", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "generate_full_name", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateState { @@ -2712,14 +2712,14 @@ export class GenerateStreetAddress extends Message { */ export class GenerateStringPhoneNumber extends Message { /** - * @generated from field: int64 min = 2; + * @generated from field: optional int64 min = 2; */ - min = protoInt64.zero; + min?: bigint; /** - * @generated from field: int64 max = 3; + * @generated from field: optional int64 max = 3; */ - max = protoInt64.zero; + max?: bigint; constructor(data?: PartialMessage) { super(); @@ -2729,8 +2729,8 @@ export class GenerateStringPhoneNumber extends Message [ - { no: 2, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 3, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 2, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 3, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateStringPhoneNumber { @@ -2755,14 +2755,14 @@ export class GenerateStringPhoneNumber extends Message { /** - * @generated from field: int64 min = 1; + * @generated from field: optional int64 min = 1; */ - min = protoInt64.zero; + min?: bigint; /** - * @generated from field: int64 max = 2; + * @generated from field: optional int64 max = 2; */ - max = protoInt64.zero; + max?: bigint; constructor(data?: PartialMessage) { super(); @@ -2772,8 +2772,8 @@ export class GenerateString extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateString"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 2, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 1, name: "min", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 2, name: "max", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateString { @@ -2891,9 +2891,9 @@ export class GenerateUtcTimestamp extends Message { */ export class GenerateUuid extends Message { /** - * @generated from field: bool include_hyphens = 1; + * @generated from field: optional bool include_hyphens = 1; */ - includeHyphens = false; + includeHyphens?: boolean; constructor(data?: PartialMessage) { super(); @@ -2903,7 +2903,7 @@ export class GenerateUuid extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateUuid"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "include_hyphens", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "include_hyphens", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateUuid { @@ -2959,9 +2959,9 @@ export class GenerateZipcode extends Message { */ export class TransformE164PhoneNumber extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -2971,7 +2971,7 @@ export class TransformE164PhoneNumber extends Message static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformE164PhoneNumber"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformE164PhoneNumber { @@ -2996,9 +2996,9 @@ export class TransformE164PhoneNumber extends Message */ export class TransformFirstName extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3008,7 +3008,7 @@ export class TransformFirstName extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformFirstName"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformFirstName { @@ -3033,14 +3033,14 @@ export class TransformFirstName extends Message { */ export class TransformFloat64 extends Message { /** - * @generated from field: double randomization_range_min = 1; + * @generated from field: optional double randomization_range_min = 1; */ - randomizationRangeMin = 0; + randomizationRangeMin?: number; /** - * @generated from field: double randomization_range_max = 2; + * @generated from field: optional double randomization_range_max = 2; */ - randomizationRangeMax = 0; + randomizationRangeMax?: number; constructor(data?: PartialMessage) { super(); @@ -3050,8 +3050,8 @@ export class TransformFloat64 extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformFloat64"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "randomization_range_min", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, - { no: 2, name: "randomization_range_max", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + { no: 1, name: "randomization_range_min", kind: "scalar", T: 1 /* ScalarType.DOUBLE */, opt: true }, + { no: 2, name: "randomization_range_max", kind: "scalar", T: 1 /* ScalarType.DOUBLE */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformFloat64 { @@ -3076,9 +3076,9 @@ export class TransformFloat64 extends Message { */ export class TransformFullName extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3088,7 +3088,7 @@ export class TransformFullName extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformFullName"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformFullName { @@ -3113,9 +3113,9 @@ export class TransformFullName extends Message { */ export class TransformInt64PhoneNumber extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3125,7 +3125,7 @@ export class TransformInt64PhoneNumber extends Message [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformInt64PhoneNumber { @@ -3150,14 +3150,14 @@ export class TransformInt64PhoneNumber extends Message { /** - * @generated from field: int64 randomization_range_min = 1; + * @generated from field: optional int64 randomization_range_min = 1; */ - randomizationRangeMin = protoInt64.zero; + randomizationRangeMin?: bigint; /** - * @generated from field: int64 randomization_range_max = 2; + * @generated from field: optional int64 randomization_range_max = 2; */ - randomizationRangeMax = protoInt64.zero; + randomizationRangeMax?: bigint; constructor(data?: PartialMessage) { super(); @@ -3167,8 +3167,8 @@ export class TransformInt64 extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformInt64"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "randomization_range_min", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 2, name: "randomization_range_max", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 1, name: "randomization_range_min", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 2, name: "randomization_range_max", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformInt64 { @@ -3193,9 +3193,9 @@ export class TransformInt64 extends Message { */ export class TransformLastName extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3205,7 +3205,7 @@ export class TransformLastName extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformLastName"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformLastName { @@ -3230,9 +3230,9 @@ export class TransformLastName extends Message { */ export class TransformPhoneNumber extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3242,7 +3242,7 @@ export class TransformPhoneNumber extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformPhoneNumber"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformPhoneNumber { @@ -3267,9 +3267,9 @@ export class TransformPhoneNumber extends Message { */ export class TransformString extends Message { /** - * @generated from field: bool preserve_length = 1; + * @generated from field: optional bool preserve_length = 1; */ - preserveLength = false; + preserveLength?: boolean; constructor(data?: PartialMessage) { super(); @@ -3279,7 +3279,7 @@ export class TransformString extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.TransformString"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "preserve_length", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TransformString { @@ -3522,9 +3522,9 @@ export class ValidateUserJavascriptCodeResponse extends Message { /** - * @generated from field: string categories = 1; + * @generated from field: optional string categories = 1; */ - categories = ""; + categories?: string; constructor(data?: PartialMessage) { super(); @@ -3534,7 +3534,7 @@ export class GenerateCategorical extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateCategorical"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "categories", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 1, name: "categories", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateCategorical { @@ -3719,9 +3719,9 @@ export class GenerateCountry extends Message { /** * An option to return the full country name of the randomly selected country or return the default of a 2-letter country code. * - * @generated from field: bool generate_full_name = 1; + * @generated from field: optional bool generate_full_name = 1; */ - generateFullName = false; + generateFullName?: boolean; constructor(data?: PartialMessage) { super(); @@ -3731,7 +3731,7 @@ export class GenerateCountry extends Message { static readonly runtime: typeof proto3 = proto3; static readonly typeName = "mgmt.v1alpha1.GenerateCountry"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "generate_full_name", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 1, name: "generate_full_name", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): GenerateCountry { diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 3530324278..39f79dd5c3 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -101,10 +101,9 @@ import ( {{ end }} {{- if eq .HasSeedParam true}} "github.com/nucleuscloud/neosync/worker/pkg/rng" - {{ end }} - {{- if gt (len .FunctInfo.Params) 0 -}} transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" - {{- end }} + {{ end }} + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" ) type {{.StructName}} struct{} @@ -128,6 +127,23 @@ func New{{.StructName}}() *{{.StructName}} { return &{{.StructName}}{} } +func New{{.StructName}}OptsFromConfig(config *mgmtv1alpha1.{{.StructName}})(*{{.StructName}}Opts, error) { + if config == nil { + return New{{.StructName}}Opts( + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "value" }}{{ continue }}{{ end }} + nil, + {{- end }} + ) + } + return New{{.StructName}}Opts( + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "value" }}{{ continue }}{{ end }} + config.{{$param.Name}}, + {{- end }} + ) +} + func New{{.StructName}}Opts( {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "value" }}{{ continue }}{{ end }} @@ -151,7 +167,7 @@ func New{{.StructName}}Opts( } {{ else if $param.HasDefault }} {{$param.Name}} := {{$param.TypeStr}}({{$param.Default}}) - if {{$param.Name}}Arg != nil && !transformer_utils.IsZeroValue(*{{$param.Name}}Arg) { + if {{$param.Name}}Arg != nil { {{$param.Name}} = *{{$param.Name}}Arg } {{ end }} diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 4f46c13e43..45a94695d0 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -56,7 +56,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig: config := transformerConfig.GetGenerateCategoricalConfig() - opts, err := NewGenerateCategoricalOpts(&config.Categories, nil) + // var categories *string + // if config != nil && config.GetCategories() != "" { + // categories = &config.Categories + // } + opts, err := NewGenerateCategoricalOptsFromConfig(config) if err != nil { return nil, err } @@ -97,7 +101,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformInt64Config: config := transformerConfig.GetTransformInt64Config() - opts, err := NewTransformInt64Opts(&config.RandomizationRangeMin, &config.RandomizationRangeMax, nil) + opts, err := NewTransformInt64Opts(config.RandomizationRangeMin, config.RandomizationRangeMax, nil) if err != nil { return nil, err } @@ -156,8 +160,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform } var excludedDomains any = config.GetExcludedDomains() opts, err := NewTransformEmailOpts( - &config.PreserveDomain, - &config.PreserveLength, + config.PreserveDomain, + config.PreserveLength, &excludedDomains, &maxLength, nil, @@ -177,7 +181,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig: config := transformerConfig.GetGenerateCardNumberConfig() - opts, err := NewGenerateCardNumberOpts(&config.ValidLuhn, nil) + opts, err := NewGenerateCardNumberOpts(config.ValidLuhn, nil) if err != nil { return nil, err } @@ -204,10 +208,12 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: config := transformerConfig.GetGenerateE164PhoneNumberConfig() - opts, err := NewGenerateInternationalPhoneNumberOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateInternationalPhoneNumberOpts(config.Min, config.Max, nil) if err != nil { return nil, err } + fmt.Println(opts.min) + fmt.Println(opts.max) generate := NewGenerateInternationalPhoneNumber().Generate return &TransformerExecutor{ Opts: opts, @@ -232,12 +238,12 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform config := transformerConfig.GetGenerateFloat64Config() var precision *int64 if config.GetPrecision() != 0 { - precision = &config.Precision + precision = config.Precision } opts, err := NewGenerateFloat64Opts( - &config.RandomizeSign, - &config.Min, - &config.Max, + config.RandomizeSign, + config.Min, + config.Max, precision, nil, // TODO: update scale based on colInfo if available nil, @@ -281,7 +287,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateGenderConfig: config := transformerConfig.GetGenerateGenderConfig() - opts, err := NewGenerateGenderOpts(&config.Abbreviate, &maxLength, nil) + opts, err := NewGenerateGenderOpts(config.Abbreviate, &maxLength, nil) if err != nil { return nil, err } @@ -308,7 +314,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateInt64Config: config := transformerConfig.GetGenerateInt64Config() - opts, err := NewGenerateInt64Opts(&config.RandomizeSign, &config.Min, &config.Max, nil) + opts, err := NewGenerateInt64Opts(config.RandomizeSign, config.Min, config.Max, nil) if err != nil { return nil, err } @@ -388,7 +394,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig: config := transformerConfig.GetGenerateStringPhoneNumberConfig() - opts, err := NewGenerateStringPhoneNumberOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateStringPhoneNumberOpts(config.Min, config.Max, nil) if err != nil { return nil, err } @@ -402,7 +408,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOpts(&config.Min, &config.Max, nil) + opts, err := NewGenerateRandomStringOpts(config.Min, config.Max, nil) if err != nil { return nil, err } @@ -482,7 +488,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig: config := transformerConfig.GetTransformE164PhoneNumberConfig() - opts, err := NewTransformE164PhoneNumberOpts(&config.PreserveLength, nil, nil) + opts, err := NewTransformE164PhoneNumberOpts(config.PreserveLength, nil, nil) if err != nil { return nil, err } @@ -496,7 +502,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig: config := transformerConfig.GetTransformFirstNameConfig() - opts, err := NewTransformFirstNameOpts(&maxLength, &config.PreserveLength, nil) + opts, err := NewTransformFirstNameOpts(&maxLength, config.PreserveLength, nil) if err != nil { return nil, err } @@ -511,8 +517,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFloat64Config: config := transformerConfig.GetTransformFloat64Config() opts, err := NewTransformFloat64Opts( - &config.RandomizationRangeMin, - &config.RandomizationRangeMax, + config.RandomizationRangeMin, + config.RandomizationRangeMax, nil, // TODO: update precision based on colInfo if available nil, // TODO: update scale based on colInfo if available nil, @@ -530,7 +536,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig: config := transformerConfig.GetTransformInt64PhoneNumberConfig() - opts, err := NewTransformInt64PhoneNumberOpts(&config.PreserveLength, nil) + opts, err := NewTransformInt64PhoneNumberOpts(config.PreserveLength, nil) if err != nil { return nil, err } @@ -544,7 +550,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformLastNameConfig: config := transformerConfig.GetTransformLastNameConfig() - opts, err := NewTransformLastNameOpts(&maxLength, &config.PreserveLength, nil) + opts, err := NewTransformLastNameOpts(&maxLength, config.PreserveLength, nil) if err != nil { return nil, err } @@ -558,7 +564,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformStringPhoneNumberOpts(&config.PreserveLength, &maxLength, nil) + opts, err := NewTransformStringPhoneNumberOpts(config.PreserveLength, &maxLength, nil) if err != nil { return nil, err } @@ -602,7 +608,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCountryConfig: generateFullName := transformerConfig.GetGenerateCountryConfig().GenerateFullName - opts, err := NewGenerateCountryOpts(&generateFullName, nil) + opts, err := NewGenerateCountryOpts(generateFullName, nil) if err != nil { return nil, err } diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformers/transformer_initializer_test.go index d10d5e27d0..88281b40f4 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer_test.go +++ b/worker/pkg/benthos/transformers/transformer_initializer_test.go @@ -25,10 +25,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateCategoricalConfig", func(t *testing.T) { + categories := "A,B,C" config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{ GenerateCategoricalConfig: &mgmtv1alpha1.GenerateCategorical{ - Categories: "A,B,C", + Categories: &categories, }, }, } @@ -54,6 +55,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateCategoricalConfig_NilConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("GenerateBoolConfig", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateBoolConfig{}, @@ -67,10 +80,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformStringConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ TransformStringConfig: &mgmtv1alpha1.TransformString{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -101,8 +115,8 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ TransformInt64Config: &mgmtv1alpha1.TransformInt64{ - RandomizationRangeMin: rmin, - RandomizationRangeMax: rmax, + RandomizationRangeMin: &rmin, + RandomizationRangeMax: &rmax, }, }, } @@ -130,10 +144,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformFullNameConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -190,13 +205,14 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformEmailConfig", func(t *testing.T) { + preserve := true emailType := mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME invalidEmailAction := mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: true, - PreserveLength: true, + PreserveDomain: &preserve, + PreserveLength: &preserve, EmailType: &emailType, InvalidEmailAction: &invalidEmailAction, }, @@ -226,10 +242,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateCardNumberConfig", func(t *testing.T) { + valid := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{ - ValidLuhn: true, + ValidLuhn: &valid, }, }, } @@ -270,13 +287,13 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) - t.Run("GenerateE164PhoneNumberConfig", func(t *testing.T) { + t.Run("GenerateInternationalPhoneNumberConfig", func(t *testing.T) { rmin, rmax := int64(10), int64(10) config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{ - Min: rmin, - Max: rmax, + Min: &rmin, + Max: &rmax, }, }, } @@ -288,7 +305,7 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^\+\d{10}$`, result) }) - t.Run("GenerateE164PhoneNumberConfig_Empty", func(t *testing.T) { + t.Run("GenerateInternationPhoneNumberConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{}, @@ -302,6 +319,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateInternationPhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("GenerateFirstNameConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig{ @@ -324,10 +353,10 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{ GenerateFloat64Config: &mgmtv1alpha1.GenerateFloat64{ - RandomizeSign: randomizeSign, - Min: rmin, - Max: rmax, - Precision: precision, + RandomizeSign: &randomizeSign, + Min: &rmin, + Max: &rmax, + Precision: &precision, }, }, } @@ -388,10 +417,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateGenderConfig", func(t *testing.T) { + abb := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{ GenerateGenderConfig: &mgmtv1alpha1.GenerateGender{ - Abbreviate: true, + Abbreviate: &abb, }, }, } @@ -441,9 +471,9 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ - RandomizeSign: randomizeSign, - Min: rmin, - Max: rmax, + RandomizeSign: &randomizeSign, + Min: &rmin, + Max: &rmax, }, }, } @@ -520,10 +550,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateStateConfig", func(t *testing.T) { + genFullName := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{ GenerateStateConfig: &mgmtv1alpha1.GenerateState{ - GenerateFullName: true, + GenerateFullName: &genFullName, }, }, } @@ -573,8 +604,8 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{ GenerateStringPhoneNumberConfig: &mgmtv1alpha1.GenerateStringPhoneNumber{ - Min: rmin, - Max: rmax, + Min: &rmin, + Max: &rmax, }, }, } @@ -607,8 +638,8 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ GenerateStringConfig: &mgmtv1alpha1.GenerateString{ - Min: rmin, - Max: rmax, + Min: &rmin, + Max: &rmax, }, }, } @@ -682,10 +713,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateUuidConfig", func(t *testing.T) { + hyphens := false config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: false, + IncludeHyphens: &hyphens, }, }, } @@ -729,10 +761,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformE164PhoneNumberConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{ TransformE164PhoneNumberConfig: &mgmtv1alpha1.TransformE164PhoneNumber{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -763,10 +796,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformFirstNameConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ TransformFirstNameConfig: &mgmtv1alpha1.TransformFirstName{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -802,8 +836,8 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{ - RandomizationRangeMin: randomizationRangeMin, - RandomizationRangeMax: randomizationRangeMax, + RandomizationRangeMin: &randomizationRangeMin, + RandomizationRangeMax: &randomizationRangeMax, }, }, } @@ -836,10 +870,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformInt64PhoneNumberConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{ TransformInt64PhoneNumberConfig: &mgmtv1alpha1.TransformInt64PhoneNumber{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -873,10 +908,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformLastNameConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{ TransformLastNameConfig: &mgmtv1alpha1.TransformLastName{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -907,10 +943,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("TransformPhoneNumberConfig", func(t *testing.T) { + preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{ TransformPhoneNumberConfig: &mgmtv1alpha1.TransformPhoneNumber{ - PreserveLength: true, + PreserveLength: &preserveLength, }, }, } @@ -1004,10 +1041,11 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { }) t.Run("GenerateCountryConfig", func(t *testing.T) { + genFull := true config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{ GenerateCountryConfig: &mgmtv1alpha1.GenerateCountry{ - GenerateFullName: true, + GenerateFullName: &genFull, }, }, } From 9e9784f38ce806a24baf990dfe14ca10565780ef Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 11:19:34 -0700 Subject: [PATCH 11/32] stash --- worker/pkg/benthos/transformers/transformer_initializer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 45a94695d0..f6934553cb 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -60,7 +60,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform // if config != nil && config.GetCategories() != "" { // categories = &config.Categories // } - opts, err := NewGenerateCategoricalOptsFromConfig(config) + opts, err := NewGenerateCategoricalOpts(config.Categories, nil) if err != nil { return nil, err } From 882c60aa334fe896f809bad9a05a8907adda4610 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 11:27:33 -0700 Subject: [PATCH 12/32] stash --- .../benthos/transformers/neosync_transformer_generator.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 39f79dd5c3..67c0fc7947 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -139,7 +139,8 @@ func New{{.StructName}}OptsFromConfig(config *mgmtv1alpha1.{{.StructName}})(*{{. return New{{.StructName}}Opts( {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "value" }}{{ continue }}{{ end }} - config.{{$param.Name}}, + {{- if eq $param.Name "seed" }} nil, {{ continue }}{{ end }} + config.{{ capitalizeFirst $param.Name }}, {{- end }} ) } @@ -265,7 +266,9 @@ func generateCode(pkgName string, funcInfo *transformers.BenthosSpec) (string, e data.ImportFmt = true } } - t := template.Must(template.New("neosyncTransformerImpl").Parse(codeTemplate)) + t := template.Must(template.New("neosyncTransformerImpl").Funcs(template.FuncMap{ + "capitalizeFirst": capitalizeFirst, + }).Parse(codeTemplate)) var out bytes.Buffer err := t.Execute(&out, data) if err != nil { From 066c887a6261891bff3d40a400f4522e417018d0 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 11:30:08 -0700 Subject: [PATCH 13/32] stash --- .../benthos/transformers/neosync_transformer_generator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 67c0fc7947..8c8bfc08f9 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -138,9 +138,9 @@ func New{{.StructName}}OptsFromConfig(config *mgmtv1alpha1.{{.StructName}})(*{{. } return New{{.StructName}}Opts( {{- range $index, $param := .FunctInfo.Params }} - {{- if eq $param.Name "value" }}{{ continue }}{{ end }} - {{- if eq $param.Name "seed" }} nil, {{ continue }}{{ end }} - config.{{ capitalizeFirst $param.Name }}, + {{- if eq $param.Name "value" }}{{ continue }}{{ end -}} + {{ if eq $param.Name "seed" }} nil, {{ continue }}{{ end }} + config.{{ capitalizeFirst $param.Name }}, {{- end }} ) } From dc40cb97dc18feccc8f109feb4eb1073e692707d Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 14:03:57 -0700 Subject: [PATCH 14/32] fixed transformer initializer --- .../benthos/transformers/gen_generate_bool.go | 1 + .../transformers/gen_generate_card_number.go | 3 +- .../transformers/gen_generate_categorical.go | 3 +- .../benthos/transformers/gen_generate_city.go | 3 +- .../transformers/gen_generate_country.go | 3 +- .../transformers/gen_generate_email.go | 5 +- .../transformers/gen_generate_first_name.go | 3 +- .../transformers/gen_generate_float.go | 7 +- .../transformers/gen_generate_full_address.go | 3 +- .../transformers/gen_generate_full_name.go | 3 +- .../transformers/gen_generate_gender.go | 5 +- .../transformers/gen_generate_int64.go | 7 +- .../gen_generate_int64_phone_number.go | 1 + ...gen_generate_international_phone_number.go | 5 +- .../transformers/gen_generate_last_name.go | 3 +- .../gen_generate_random_string.go | 5 +- .../benthos/transformers/gen_generate_ssn.go | 1 + .../transformers/gen_generate_state.go | 3 +- .../gen_generate_street_address.go | 3 +- .../gen_generate_string_phone_number.go | 5 +- .../gen_generate_unix_timestamp.go | 1 + .../transformers/gen_generate_username.go | 3 +- .../gen_generate_utc_timestamp.go | 1 + .../benthos/transformers/gen_generate_uuid.go | 4 +- .../transformers/gen_generate_zipcode.go | 1 + .../gen_transform_character_scramble.go | 1 + .../gen_transform_e164_phone_number.go | 5 +- .../transformers/gen_transform_email.go | 13 +- .../transformers/gen_transform_first_name.go | 5 +- .../transformers/gen_transform_float.go | 5 +- .../transformers/gen_transform_full_name.go | 5 +- .../transformers/gen_transform_int64.go | 5 +- .../gen_transform_int64_phone_number.go | 3 +- .../transformers/gen_transform_lastname.go | 5 +- .../transformers/gen_transform_string.go | 7 +- .../gen_transform_string_phone_number.go | 5 +- .../pkg/benthos/transformers/generate_bool.go | 5 + .../transformers/generate_card_number.go | 8 + .../transformers/generate_categorical.go | 8 + .../pkg/benthos/transformers/generate_city.go | 13 + .../benthos/transformers/generate_country.go | 13 + .../benthos/transformers/generate_email.go | 16 + .../transformers/generate_first_name.go | 10 + .../benthos/transformers/generate_float.go | 15 + .../transformers/generate_full_address.go | 13 + .../transformers/generate_full_name.go | 13 + .../benthos/transformers/generate_gender.go | 15 + .../benthos/transformers/generate_int64.go | 17 + .../generate_int64_phone_number.go | 5 + .../generate_international_phone_number.go | 16 + .../transformers/generate_last_name.go | 13 + .../transformers/generate_random_string.go | 16 + .../transformers/generate_sha256hash.go | 5 + .../pkg/benthos/transformers/generate_ssn.go | 5 + .../benthos/transformers/generate_state.go | 8 + .../transformers/generate_street_address.go | 10 + .../generate_string_phone_number.go | 8 + .../transformers/generate_unix_timestamp.go | 4 + .../benthos/transformers/generate_username.go | 9 + .../transformers/generate_utc_timestamp.go | 5 + .../pkg/benthos/transformers/generate_uuid.go | 8 + .../benthos/transformers/generate_zipcode.go | 5 + .../neosync_transformer_generator.go | 19 - .../transform_character_scramble.go | 8 + .../transform_e164_phone_number.go | 8 + .../benthos/transformers/transform_email.go | 49 ++ .../transformers/transform_first_name.go | 12 + .../benthos/transformers/transform_float.go | 14 + .../transformers/transform_full_name.go | 12 + .../benthos/transformers/transform_int64.go | 12 + .../transform_int64_phone_number.go | 8 + .../transformers/transform_lastname.go | 12 + .../benthos/transformers/transform_string.go | 13 + .../transform_string_phone_number.go | 11 + .../transformers/transformer_initializer.go | 167 ++---- .../transformer_initializer_test.go | 501 +++++++++++++++++- 76 files changed, 1066 insertions(+), 184 deletions(-) diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index 2bdd40be61..f01534234b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateBool struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index 819103523b..442343d5f4 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateCardNumber struct{} @@ -28,7 +29,7 @@ func NewGenerateCardNumberOpts( seedArg *int64, ) (*GenerateCardNumberOpts, error) { validLuhn := bool(false) - if validLuhnArg != nil && !transformer_utils.IsZeroValue(*validLuhnArg) { + if validLuhnArg != nil { validLuhn = *validLuhnArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index e48a35a859..5be1dfd5c9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateCategorical struct{} @@ -28,7 +29,7 @@ func NewGenerateCategoricalOpts( seedArg *int64, ) (*GenerateCategoricalOpts, error) { categories := string("ultimo,proximo,semper") - if categoriesArg != nil && !transformer_utils.IsZeroValue(*categoriesArg) { + if categoriesArg != nil { categories = *categoriesArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index 975fcf0f17..694256f7ab 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateCity struct{} @@ -28,7 +29,7 @@ func NewGenerateCityOpts( seedArg *int64, ) (*GenerateCityOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index df69dc011b..e14b9da9e9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateCountry struct{} @@ -28,7 +29,7 @@ func NewGenerateCountryOpts( seedArg *int64, ) (*GenerateCountryOpts, error) { generateFullName := bool(false) - if generateFullNameArg != nil && !transformer_utils.IsZeroValue(*generateFullNameArg) { + if generateFullNameArg != nil { generateFullName = *generateFullNameArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index 396a864db6..17c24a8921 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateEmail struct{} @@ -30,12 +31,12 @@ func NewGenerateEmailOpts( seedArg *int64, ) (*GenerateEmailOpts, error) { maxLength := int64(100000) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } emailType := string(GenerateEmailType_UuidV4.String()) - if emailTypeArg != nil && !transformer_utils.IsZeroValue(*emailTypeArg) { + if emailTypeArg != nil { emailType = *emailTypeArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index 21c4c924c1..897b4ec67f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateFirstName struct{} @@ -28,7 +29,7 @@ func NewGenerateFirstNameOpts( seedArg *int64, ) (*GenerateFirstNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index 89b2ba6875..e230e6c05e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateFloat64 struct{} @@ -36,17 +37,17 @@ func NewGenerateFloat64Opts( seedArg *int64, ) (*GenerateFloat64Opts, error) { randomizeSign := bool(false) - if randomizeSignArg != nil && !transformer_utils.IsZeroValue(*randomizeSignArg) { + if randomizeSignArg != nil { randomizeSign = *randomizeSignArg } min := float64(1) - if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + if minArg != nil { min = *minArg } max := float64(10000) - if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + if maxArg != nil { max = *maxArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index 1e7bf1ce4b..c594d3db52 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateFullAddress struct{} @@ -28,7 +29,7 @@ func NewGenerateFullAddressOpts( seedArg *int64, ) (*GenerateFullAddressOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index 20a9693908..3913f1ed25 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateFullName struct{} @@ -28,7 +29,7 @@ func NewGenerateFullNameOpts( seedArg *int64, ) (*GenerateFullNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index c7d63008ac..cc2cfabdcf 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateGender struct{} @@ -30,12 +31,12 @@ func NewGenerateGenderOpts( seedArg *int64, ) (*GenerateGenderOpts, error) { abbreviate := bool(false) - if abbreviateArg != nil && !transformer_utils.IsZeroValue(*abbreviateArg) { + if abbreviateArg != nil { abbreviate = *abbreviateArg } maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index 07736c9d22..2819cf0951 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateInt64 struct{} @@ -32,17 +33,17 @@ func NewGenerateInt64Opts( seedArg *int64, ) (*GenerateInt64Opts, error) { randomizeSign := bool(false) - if randomizeSignArg != nil && !transformer_utils.IsZeroValue(*randomizeSignArg) { + if randomizeSignArg != nil { randomizeSign = *randomizeSignArg } min := int64(1) - if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + if minArg != nil { min = *minArg } max := int64(10000) - if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + if maxArg != nil { max = *maxArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index c2fb581bfb..9bda577920 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateInt64PhoneNumber struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index 693b6eee35..3227f39c06 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateInternationalPhoneNumber struct{} @@ -30,12 +31,12 @@ func NewGenerateInternationalPhoneNumberOpts( seedArg *int64, ) (*GenerateInternationalPhoneNumberOpts, error) { min := int64(9) - if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + if minArg != nil { min = *minArg } max := int64(15) - if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + if maxArg != nil { max = *maxArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index a727585cd3..627aa17caf 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateLastName struct{} @@ -28,7 +29,7 @@ func NewGenerateLastNameOpts( seedArg *int64, ) (*GenerateLastNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index 9971041761..c7a51adf2f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateRandomString struct{} @@ -30,12 +31,12 @@ func NewGenerateRandomStringOpts( seedArg *int64, ) (*GenerateRandomStringOpts, error) { min := int64(1) - if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + if minArg != nil { min = *minArg } max := int64(100) - if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + if maxArg != nil { max = *maxArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index 9a32ff4464..e8713dff7e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateSSN struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index a171d165b7..3a6cbb432e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateState struct{} @@ -28,7 +29,7 @@ func NewGenerateStateOpts( seedArg *int64, ) (*GenerateStateOpts, error) { generateFullName := bool(false) - if generateFullNameArg != nil && !transformer_utils.IsZeroValue(*generateFullNameArg) { + if generateFullNameArg != nil { generateFullName = *generateFullNameArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index 3d5acc9dd6..1d96bb09a2 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateStreetAddress struct{} @@ -28,7 +29,7 @@ func NewGenerateStreetAddressOpts( seedArg *int64, ) (*GenerateStreetAddressOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index f263bf0c08..1086da46a6 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateStringPhoneNumber struct{} @@ -30,12 +31,12 @@ func NewGenerateStringPhoneNumberOpts( seedArg *int64, ) (*GenerateStringPhoneNumberOpts, error) { min := int64(9) - if minArg != nil && !transformer_utils.IsZeroValue(*minArg) { + if minArg != nil { min = *minArg } max := int64(15) - if maxArg != nil && !transformer_utils.IsZeroValue(*maxArg) { + if maxArg != nil { max = *maxArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index e9c592b087..6ebf9e5ace 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateUnixTimestamp struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index 2893408d73..ac2ad8bf3a 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateUsername struct{} @@ -28,7 +29,7 @@ func NewGenerateUsernameOpts( seedArg *int64, ) (*GenerateUsernameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index d2383feed4..86713d2c51 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateUTCTimestamp struct{} diff --git a/worker/pkg/benthos/transformers/gen_generate_uuid.go b/worker/pkg/benthos/transformers/gen_generate_uuid.go index 741c847386..3c6801badb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_uuid.go +++ b/worker/pkg/benthos/transformers/gen_generate_uuid.go @@ -4,7 +4,7 @@ package transformers -import (transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" +import ( ) type GenerateUUID struct{} @@ -21,7 +21,7 @@ func NewGenerateUUIDOpts( includeHyphensArg *bool, ) (*GenerateUUIDOpts, error) { includeHyphens := bool(true) - if includeHyphensArg != nil && !transformer_utils.IsZeroValue(*includeHyphensArg) { + if includeHyphensArg != nil { includeHyphens = *includeHyphensArg } diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index 62f0d8ff96..4fb6c7705e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type GenerateZipcode struct{} diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 91d646fe98..8422756295 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformCharacterScramble struct{} diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index 280907faff..1d3b709aef 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformE164PhoneNumber struct{} @@ -30,12 +31,12 @@ func NewTransformE164PhoneNumberOpts( seedArg *int64, ) (*TransformE164PhoneNumberOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } maxLength := int64(15) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index 22d92045e8..1c9c09df11 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformEmail struct{} @@ -38,22 +39,22 @@ func NewTransformEmailOpts( invalidEmailActionArg *string, ) (*TransformEmailOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } preserveDomain := bool(false) - if preserveDomainArg != nil && !transformer_utils.IsZeroValue(*preserveDomainArg) { + if preserveDomainArg != nil { preserveDomain = *preserveDomainArg } excludedDomains := any([]any{}) - if excludedDomainsArg != nil && !transformer_utils.IsZeroValue(*excludedDomainsArg) { + if excludedDomainsArg != nil { excludedDomains = *excludedDomainsArg } maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -63,12 +64,12 @@ func NewTransformEmailOpts( } emailType := string(GenerateEmailType_UuidV4.String()) - if emailTypeArg != nil && !transformer_utils.IsZeroValue(*emailTypeArg) { + if emailTypeArg != nil { emailType = *emailTypeArg } invalidEmailAction := string(InvalidEmailAction_Reject.String()) - if invalidEmailActionArg != nil && !transformer_utils.IsZeroValue(*invalidEmailActionArg) { + if invalidEmailActionArg != nil { invalidEmailAction = *invalidEmailActionArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index 9aee9f9bef..29e17b7959 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformFirstName struct{} @@ -30,12 +31,12 @@ func NewTransformFirstNameOpts( seedArg *int64, ) (*TransformFirstNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index 0d98ab2329..e73d8a8425 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformFloat64 struct{} @@ -34,12 +35,12 @@ func NewTransformFloat64Opts( seedArg *int64, ) (*TransformFloat64Opts, error) { randomizationRangeMin := float64(1) - if randomizationRangeMinArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMinArg) { + if randomizationRangeMinArg != nil { randomizationRangeMin = *randomizationRangeMinArg } randomizationRangeMax := float64(10000) - if randomizationRangeMaxArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMaxArg) { + if randomizationRangeMaxArg != nil { randomizationRangeMax = *randomizationRangeMaxArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index b05327b280..9c188facb7 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformFullName struct{} @@ -30,12 +31,12 @@ func NewTransformFullNameOpts( seedArg *int64, ) (*TransformFullNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index c48b1068d0..8384eafa64 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformInt64 struct{} @@ -30,12 +31,12 @@ func NewTransformInt64Opts( seedArg *int64, ) (*TransformInt64Opts, error) { randomizationRangeMin := int64(1) - if randomizationRangeMinArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMinArg) { + if randomizationRangeMinArg != nil { randomizationRangeMin = *randomizationRangeMinArg } randomizationRangeMax := int64(10000) - if randomizationRangeMaxArg != nil && !transformer_utils.IsZeroValue(*randomizationRangeMaxArg) { + if randomizationRangeMaxArg != nil { randomizationRangeMax = *randomizationRangeMaxArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index 4fa62bd6df..d60ffb70cb 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformInt64PhoneNumber struct{} @@ -28,7 +29,7 @@ func NewTransformInt64PhoneNumberOpts( seedArg *int64, ) (*TransformInt64PhoneNumberOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index 9650edc1ea..4762049f33 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformLastName struct{} @@ -30,12 +31,12 @@ func NewTransformLastNameOpts( seedArg *int64, ) (*TransformLastNameOpts, error) { maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index 9ad7bddb03..8d7ad4b925 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformString struct{} @@ -32,17 +33,17 @@ func NewTransformStringOpts( seedArg *int64, ) (*TransformStringOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } minLength := int64(1) - if minLengthArg != nil && !transformer_utils.IsZeroValue(*minLengthArg) { + if minLengthArg != nil { minLength = *minLengthArg } maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 726ff6a619..70b09bfe1e 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -9,6 +9,7 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + ) type TransformStringPhoneNumber struct{} @@ -30,12 +31,12 @@ func NewTransformStringPhoneNumberOpts( seedArg *int64, ) (*TransformStringPhoneNumberOpts, error) { preserveLength := bool(false) - if preserveLengthArg != nil && !transformer_utils.IsZeroValue(*preserveLengthArg) { + if preserveLengthArg != nil { preserveLength = *preserveLengthArg } maxLength := int64(100) - if maxLengthArg != nil && !transformer_utils.IsZeroValue(*maxLengthArg) { + if maxLengthArg != nil { maxLength = *maxLengthArg } diff --git a/worker/pkg/benthos/transformers/generate_bool.go b/worker/pkg/benthos/transformers/generate_bool.go index 1ca1fe3b46..c7816fa749 100644 --- a/worker/pkg/benthos/transformers/generate_bool.go +++ b/worker/pkg/benthos/transformers/generate_bool.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -36,6 +37,10 @@ func init() { } } +func NewGenerateBoolOptsFromConfig(config *mgmtv1alpha1.GenerateBool) (*GenerateBoolOpts, error) { + return NewGenerateBoolOpts(nil) +} + func (t *GenerateBool) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateBoolOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_card_number.go b/worker/pkg/benthos/transformers/generate_card_number.go index d851c7a2c1..c1f6e7796a 100644 --- a/worker/pkg/benthos/transformers/generate_card_number.go +++ b/worker/pkg/benthos/transformers/generate_card_number.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -52,6 +53,13 @@ func init() { } } +func NewGenerateCardNumberOptsFromConfig(config *mgmtv1alpha1.GenerateCardNumber) (*GenerateCardNumberOpts, error) { + if config == nil { + return NewGenerateCardNumberOpts(nil, nil) + } + return NewGenerateCardNumberOpts(config.ValidLuhn, nil) +} + func (t *GenerateCardNumber) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateCardNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_categorical.go b/worker/pkg/benthos/transformers/generate_categorical.go index 3a6d01e508..5e69035d63 100644 --- a/worker/pkg/benthos/transformers/generate_categorical.go +++ b/worker/pkg/benthos/transformers/generate_categorical.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -46,6 +47,13 @@ func init() { } } +func NewGenerateCategoricalOptsFromConfig(config *mgmtv1alpha1.GenerateCategorical) (*GenerateCategoricalOpts, error) { + if config == nil { + return NewGenerateCategoricalOpts(nil, nil) + } + return NewGenerateCategoricalOpts(config.Categories, nil) +} + func (t *GenerateCategorical) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateCategoricalOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_city.go b/worker/pkg/benthos/transformers/generate_city.go index 799cfe27a8..120ec4ab67 100644 --- a/worker/pkg/benthos/transformers/generate_city.go +++ b/worker/pkg/benthos/transformers/generate_city.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -47,6 +48,18 @@ func init() { } } +func NewGenerateCityOptsFromConfig(config *mgmtv1alpha1.GenerateCity, maxLength *int64) (*GenerateCityOpts, error) { + if config == nil { + return NewGenerateCityOpts( + nil, + nil, + ) + } + return NewGenerateCityOpts( + maxLength, nil, + ) +} + func (t *GenerateCity) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateCityOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_country.go b/worker/pkg/benthos/transformers/generate_country.go index 12a2fad51c..c409470aed 100644 --- a/worker/pkg/benthos/transformers/generate_country.go +++ b/worker/pkg/benthos/transformers/generate_country.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -41,6 +42,18 @@ func init() { } } +func NewGenerateCountryOptsFromConfig(config *mgmtv1alpha1.GenerateCountry) (*GenerateCountryOpts, error) { + if config == nil { + return NewGenerateCountryOpts( + nil, + nil, + ) + } + return NewGenerateCountryOpts( + config.GenerateFullName, nil, + ) +} + func (t *GenerateCountry) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateCountryOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_email.go b/worker/pkg/benthos/transformers/generate_email.go index c3fa8056c7..5b1670fc99 100644 --- a/worker/pkg/benthos/transformers/generate_email.go +++ b/worker/pkg/benthos/transformers/generate_email.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/google/uuid" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -75,6 +76,21 @@ func init() { } } +func NewGenerateEmailOptsFromConfig(config *mgmtv1alpha1.GenerateEmail, maxLength *int64) (*GenerateEmailOpts, error) { + if config == nil { + return NewGenerateEmailOpts(nil, nil, nil) + } + var emailType *string + if config.EmailType != nil { + emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() + emailType = &emailTypeStr + } + return NewGenerateEmailOpts( + maxLength, + emailType, nil, + ) +} + func (t *GenerateEmail) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateEmailOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_first_name.go b/worker/pkg/benthos/transformers/generate_first_name.go index 64cd647027..910f44184c 100644 --- a/worker/pkg/benthos/transformers/generate_first_name.go +++ b/worker/pkg/benthos/transformers/generate_first_name.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -47,6 +48,15 @@ func init() { } } +func NewGenerateFirstNameOptsFromConfig(config *mgmtv1alpha1.GenerateFirstName, maxLength *int64) (*GenerateFirstNameOpts, error) { + if config == nil { + return NewGenerateFirstNameOpts(nil, nil) + } + return NewGenerateFirstNameOpts( + maxLength, nil, + ) +} + func (t *GenerateFirstName) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateFirstNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_float.go b/worker/pkg/benthos/transformers/generate_float.go index 90f2249e01..d1e348bf88 100644 --- a/worker/pkg/benthos/transformers/generate_float.go +++ b/worker/pkg/benthos/transformers/generate_float.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -72,6 +73,20 @@ func init() { } } +func NewGenerateFloat64OptsFromConfig(config *mgmtv1alpha1.GenerateFloat64, scale *int64) (*GenerateFloat64Opts, error) { + if config == nil { + return NewGenerateFloat64Opts(nil, nil, nil, nil, nil, nil) + } + return NewGenerateFloat64Opts( + config.RandomizeSign, + config.Min, + config.Max, + config.Precision, + nil, + nil, + ) +} + func (t *GenerateFloat64) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateFloat64Opts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_full_address.go b/worker/pkg/benthos/transformers/generate_full_address.go index 5374f8fe68..d773cff6b2 100644 --- a/worker/pkg/benthos/transformers/generate_full_address.go +++ b/worker/pkg/benthos/transformers/generate_full_address.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -47,6 +48,18 @@ func init() { } } +func NewGenerateFullAddressOptsFromConfig(config *mgmtv1alpha1.GenerateFullAddress, maxLength *int64) (*GenerateFullAddressOpts, error) { + if config == nil { + return NewGenerateFullAddressOpts( + nil, + nil, + ) + } + return NewGenerateFullAddressOpts( + maxLength, nil, + ) +} + func (t *GenerateFullAddress) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateFullAddressOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_full_name.go b/worker/pkg/benthos/transformers/generate_full_name.go index f7216329a4..9541d1d2db 100644 --- a/worker/pkg/benthos/transformers/generate_full_name.go +++ b/worker/pkg/benthos/transformers/generate_full_name.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -47,6 +48,18 @@ func init() { } } +func NewGenerateFullNameOptsFromConfig(config *mgmtv1alpha1.GenerateFullName, maxLength *int64) (*GenerateFullNameOpts, error) { + if config == nil { + return NewGenerateFullNameOpts( + nil, + nil, + ) + } + return NewGenerateFullNameOpts( + maxLength, nil, + ) +} + func (t *GenerateFullName) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateFullNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_gender.go b/worker/pkg/benthos/transformers/generate_gender.go index e5313f56b4..2c6010aadc 100644 --- a/worker/pkg/benthos/transformers/generate_gender.go +++ b/worker/pkg/benthos/transformers/generate_gender.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -49,6 +50,20 @@ func init() { } } +func NewGenerateGenderOptsFromConfig(config *mgmtv1alpha1.GenerateGender, maxLength *int64) (*GenerateGenderOpts, error) { + if config == nil { + return NewGenerateGenderOpts( + nil, + nil, + nil, + ) + } + return NewGenerateGenderOpts( + config.Abbreviate, + maxLength, nil, + ) +} + func (t *GenerateGender) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateGenderOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_int64.go b/worker/pkg/benthos/transformers/generate_int64.go index 6603e68988..31b0451c40 100644 --- a/worker/pkg/benthos/transformers/generate_int64.go +++ b/worker/pkg/benthos/transformers/generate_int64.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -76,6 +77,22 @@ func init() { } } +func NewGenerateInt64OptsFromConfig(config *mgmtv1alpha1.GenerateInt64) (*GenerateInt64Opts, error) { + if config == nil { + return NewGenerateInt64Opts( + nil, + nil, + nil, + nil, + ) + } + return NewGenerateInt64Opts( + config.RandomizeSign, + config.Min, + config.Max, nil, + ) +} + func (t *GenerateInt64) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateInt64Opts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_int64_phone_number.go b/worker/pkg/benthos/transformers/generate_int64_phone_number.go index 688ac3a10c..2e4dc359ee 100644 --- a/worker/pkg/benthos/transformers/generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_int64_phone_number.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +45,10 @@ func init() { } } +func NewGenerateInt64PhoneNumberOptsFromConfig(config *mgmtv1alpha1.GenerateInt64PhoneNumber) (*GenerateInt64PhoneNumberOpts, error) { + return NewGenerateInt64PhoneNumberOpts(nil) +} + func (t *GenerateInt64PhoneNumber) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateInt64PhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_international_phone_number.go b/worker/pkg/benthos/transformers/generate_international_phone_number.go index 715f48cc17..f10a4a8c62 100644 --- a/worker/pkg/benthos/transformers/generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_international_phone_number.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -56,6 +57,21 @@ func init() { } } +func NewGenerateInternationalPhoneNumberOptsFromConfig(config *mgmtv1alpha1.GenerateE164PhoneNumber) (*GenerateInternationalPhoneNumberOpts, error) { + if config == nil { + return NewGenerateInternationalPhoneNumberOpts( + nil, + nil, + nil, + ) + } + return NewGenerateInternationalPhoneNumberOpts( + config.Min, + config.Max, + nil, + ) +} + func (t *GenerateInternationalPhoneNumber) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateInternationalPhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_last_name.go b/worker/pkg/benthos/transformers/generate_last_name.go index 749d7536dd..a971ffd74d 100644 --- a/worker/pkg/benthos/transformers/generate_last_name.go +++ b/worker/pkg/benthos/transformers/generate_last_name.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -47,6 +48,18 @@ func init() { } } +func NewGenerateLastNameOptsFromConfig(config *mgmtv1alpha1.GenerateLastName, maxLength *int64) (*GenerateLastNameOpts, error) { + if config == nil { + return NewGenerateLastNameOpts( + nil, + nil, + ) + } + return NewGenerateLastNameOpts( + maxLength, nil, + ) +} + func (t *GenerateLastName) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateLastNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_random_string.go index 69b1bdad36..16bd89643b 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -54,6 +55,21 @@ func init() { } } +func NewGenerateRandomStringOptsFromConfig(config *mgmtv1alpha1.GenerateString) (*GenerateRandomStringOpts, error) { + if config == nil { + return NewGenerateRandomStringOpts( + nil, + nil, + nil, + ) + } + return NewGenerateRandomStringOpts( + config.Min, + config.Max, + nil, + ) +} + func (t *GenerateRandomString) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateRandomStringOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_sha256hash.go b/worker/pkg/benthos/transformers/generate_sha256hash.go index 640af643e5..8123c2db37 100644 --- a/worker/pkg/benthos/transformers/generate_sha256hash.go +++ b/worker/pkg/benthos/transformers/generate_sha256hash.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/google/uuid" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/warpstreamlabs/bento/public/bloblang" ) @@ -28,6 +29,10 @@ func init() { } } +func NewGenerateSHA256HashOptsFromConfig(config *mgmtv1alpha1.GenerateSha256Hash) (*GenerateSHA256HashOpts, error) { + return NewGenerateSHA256HashOpts() +} + func (t *GenerateSHA256Hash) Generate(opts any) (any, error) { return generateRandomSHA256Hash(uuid.NewString()) } diff --git a/worker/pkg/benthos/transformers/generate_ssn.go b/worker/pkg/benthos/transformers/generate_ssn.go index 25ef097699..9eb9eb70b5 100644 --- a/worker/pkg/benthos/transformers/generate_ssn.go +++ b/worker/pkg/benthos/transformers/generate_ssn.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -36,6 +37,10 @@ func init() { } } +func NewGenerateSSNOptsFromConfig(config *mgmtv1alpha1.GenerateSSN) (*GenerateSSNOpts, error) { + return NewGenerateSSNOpts(nil) +} + func (t *GenerateSSN) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateSSNOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_state.go b/worker/pkg/benthos/transformers/generate_state.go index 214582057e..23537597c0 100644 --- a/worker/pkg/benthos/transformers/generate_state.go +++ b/worker/pkg/benthos/transformers/generate_state.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -41,6 +42,13 @@ func init() { } } +func NewGenerateStateOptsFromConfig(config *mgmtv1alpha1.GenerateState) (*GenerateStateOpts, error) { + if config == nil { + return NewGenerateStateOpts(nil, nil) + } + return NewGenerateStateOpts(config.GenerateFullName, nil) +} + func (t *GenerateState) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateStateOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_street_address.go b/worker/pkg/benthos/transformers/generate_street_address.go index 9e9070ef0f..6edd3c0469 100644 --- a/worker/pkg/benthos/transformers/generate_street_address.go +++ b/worker/pkg/benthos/transformers/generate_street_address.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -55,6 +56,15 @@ func init() { } } +func NewGenerateStreetAddressOptsFromConfig(config *mgmtv1alpha1.GenerateStreetAddress, maxLength *int64) (*GenerateStreetAddressOpts, error) { + if config == nil { + return NewGenerateStreetAddressOpts(nil, nil) + } + return NewGenerateStreetAddressOpts( + maxLength, nil, + ) +} + func (t *GenerateStreetAddress) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateStreetAddressOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_string_phone_number.go b/worker/pkg/benthos/transformers/generate_string_phone_number.go index afdefeef6d..8d13247ae1 100644 --- a/worker/pkg/benthos/transformers/generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/generate_string_phone_number.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -54,6 +55,13 @@ func init() { } } +func NewGenerateStringPhoneNumberOptsFromConfig(config *mgmtv1alpha1.GenerateStringPhoneNumber) (*GenerateStringPhoneNumberOpts, error) { + if config == nil { + return NewGenerateStringPhoneNumberOpts(nil, nil, nil) + } + return NewGenerateStringPhoneNumberOpts(config.Min, config.Max, nil) +} + func (t *GenerateStringPhoneNumber) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateStringPhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_unix_timestamp.go b/worker/pkg/benthos/transformers/generate_unix_timestamp.go index c283313753..24c9702f88 100644 --- a/worker/pkg/benthos/transformers/generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/generate_unix_timestamp.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -35,6 +36,9 @@ func init() { panic(err) } } +func NewGenerateUnixTimestampOptsFromConfig(config *mgmtv1alpha1.GenerateUnixTimestamp) (*GenerateUnixTimestampOpts, error) { + return NewGenerateUnixTimestampOpts(nil) +} func (t *GenerateUnixTimestamp) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateUnixTimestampOpts) diff --git a/worker/pkg/benthos/transformers/generate_username.go b/worker/pkg/benthos/transformers/generate_username.go index 8b0302fcd0..36d9745564 100644 --- a/worker/pkg/benthos/transformers/generate_username.go +++ b/worker/pkg/benthos/transformers/generate_username.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -46,6 +47,14 @@ func init() { panic(err) } } +func NewGenerateUsernameOptsFromConfig(config *mgmtv1alpha1.GenerateUsername, maxLength *int64) (*GenerateUsernameOpts, error) { + if config == nil { + return NewGenerateUsernameOpts(nil, nil) + } + return NewGenerateUsernameOpts( + maxLength, nil, + ) +} func (t *GenerateUsername) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateUsernameOpts) diff --git a/worker/pkg/benthos/transformers/generate_utc_timestamp.go b/worker/pkg/benthos/transformers/generate_utc_timestamp.go index d6ffb7fbb9..d751bdda83 100644 --- a/worker/pkg/benthos/transformers/generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/generate_utc_timestamp.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -36,6 +37,10 @@ func init() { } } +func NewGenerateUTCTimestampOptsFromConfig(config *mgmtv1alpha1.GenerateUtcTimestamp) (*GenerateUTCTimestampOpts, error) { + return NewGenerateUTCTimestampOpts(nil) +} + func (t *GenerateUTCTimestamp) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateUTCTimestampOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_uuid.go b/worker/pkg/benthos/transformers/generate_uuid.go index 3a0d672a9d..3ec66c28da 100644 --- a/worker/pkg/benthos/transformers/generate_uuid.go +++ b/worker/pkg/benthos/transformers/generate_uuid.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/warpstreamlabs/bento/public/bloblang" ) @@ -33,6 +34,13 @@ func init() { } } +func NewGenerateUUIDOptsFromConfig(config *mgmtv1alpha1.GenerateUuid) (*GenerateUUIDOpts, error) { + if config == nil { + return NewGenerateUUIDOpts(nil) + } + return NewGenerateUUIDOpts(config.IncludeHyphens) +} + func (t *GenerateUUID) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateUUIDOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/generate_zipcode.go b/worker/pkg/benthos/transformers/generate_zipcode.go index 2f4a55abd4..56ca8e835a 100644 --- a/worker/pkg/benthos/transformers/generate_zipcode.go +++ b/worker/pkg/benthos/transformers/generate_zipcode.go @@ -3,6 +3,7 @@ package transformers import ( "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -37,6 +38,10 @@ func init() { } } +func NewGenerateZipcodeOptsFromConfig(config *mgmtv1alpha1.GenerateZipcode) (*GenerateZipcodeOpts, error) { + return NewGenerateZipcodeOpts(nil) +} + func (t *GenerateZipcode) Generate(opts any) (any, error) { parsedOpts, ok := opts.(*GenerateZipcodeOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 8c8bfc08f9..a73eb30060 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -103,7 +103,6 @@ import ( "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" {{ end }} - mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" ) type {{.StructName}} struct{} @@ -127,24 +126,6 @@ func New{{.StructName}}() *{{.StructName}} { return &{{.StructName}}{} } -func New{{.StructName}}OptsFromConfig(config *mgmtv1alpha1.{{.StructName}})(*{{.StructName}}Opts, error) { - if config == nil { - return New{{.StructName}}Opts( - {{- range $index, $param := .FunctInfo.Params }} - {{- if eq $param.Name "value" }}{{ continue }}{{ end }} - nil, - {{- end }} - ) - } - return New{{.StructName}}Opts( - {{- range $index, $param := .FunctInfo.Params }} - {{- if eq $param.Name "value" }}{{ continue }}{{ end -}} - {{ if eq $param.Name "seed" }} nil, {{ continue }}{{ end }} - config.{{ capitalizeFirst $param.Name }}, - {{- end }} - ) -} - func New{{.StructName}}Opts( {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "value" }}{{ continue }}{{ end }} diff --git a/worker/pkg/benthos/transformers/transform_character_scramble.go b/worker/pkg/benthos/transformers/transform_character_scramble.go index b5b94c088f..dd80e75256 100644 --- a/worker/pkg/benthos/transformers/transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/transform_character_scramble.go @@ -6,6 +6,7 @@ import ( "strings" "unicode" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -68,6 +69,13 @@ func init() { } } +func NewTransformCharacterScrambleOptsFromConfig(config *mgmtv1alpha1.TransformCharacterScramble) (*TransformCharacterScrambleOpts, error) { + if config == nil { + return NewTransformCharacterScrambleOpts(nil, nil) + } + return NewTransformCharacterScrambleOpts(config.UserProvidedRegex, nil) +} + func (t *TransformCharacterScramble) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformCharacterScrambleOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_e164_phone_number.go b/worker/pkg/benthos/transformers/transform_e164_phone_number.go index 8be8ece860..47bcde55bd 100644 --- a/worker/pkg/benthos/transformers/transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_e164_phone_number.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -67,6 +68,13 @@ func init() { } } +func NewTransformE164PhoneNumberOptsFromConfig(config *mgmtv1alpha1.TransformE164PhoneNumber, maxLength *int64) (*TransformE164PhoneNumberOpts, error) { + if config == nil { + return NewTransformE164PhoneNumberOpts(nil, nil, nil) + } + return NewTransformE164PhoneNumberOpts(config.PreserveLength, maxLength, nil) +} + func (t *TransformE164PhoneNumber) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformE164PhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_email.go b/worker/pkg/benthos/transformers/transform_email.go index 10b87caa13..edb83a34a6 100644 --- a/worker/pkg/benthos/transformers/transform_email.go +++ b/worker/pkg/benthos/transformers/transform_email.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/google/uuid" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -129,6 +130,32 @@ func init() { } } +func NewTransformEmailOptsFromConfig(config *mgmtv1alpha1.TransformEmail, maxLength *int64) (*TransformEmailOpts, error) { + if config == nil { + return NewTransformEmailOpts(nil, nil, nil, nil, nil, nil, nil) + } + var emailType *string + if config.EmailType != nil { + emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() + emailType = &emailTypeStr + } + var invalidEmailAction *string + if config.InvalidEmailAction != nil { + invalidEmailActionStr := dtoInvalidEmailActionToTransformerInvalidEmailAction(config.GetInvalidEmailAction()).String() + invalidEmailAction = &invalidEmailActionStr + } + var excludedDomains any = config.GetExcludedDomains() + return NewTransformEmailOpts( + config.PreserveLength, + config.PreserveDomain, + &excludedDomains, + maxLength, + nil, + emailType, + invalidEmailAction, + ) +} + func (t *TransformEmail) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformEmailOpts) if !ok { @@ -296,3 +323,25 @@ func transformEmail( generatedemail := fmt.Sprintf("%s@%s", newname, newdomain) return &generatedemail, nil } + +func dtoEmailTypeToTransformerEmailType(dto mgmtv1alpha1.GenerateEmailType) GenerateEmailType { + switch dto { + case mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME: + return GenerateEmailType_FullName + default: + return GenerateEmailType_UuidV4 + } +} + +func dtoInvalidEmailActionToTransformerInvalidEmailAction(dto mgmtv1alpha1.InvalidEmailAction) InvalidEmailAction { + switch dto { + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE: + return InvalidEmailAction_Generate + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_NULL: + return InvalidEmailAction_Null + case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_PASSTHROUGH: + return InvalidEmailAction_Passthrough + default: + return InvalidEmailAction_Reject + } +} diff --git a/worker/pkg/benthos/transformers/transform_first_name.go b/worker/pkg/benthos/transformers/transform_first_name.go index 66fa0a6b3d..50573ce9ac 100644 --- a/worker/pkg/benthos/transformers/transform_first_name.go +++ b/worker/pkg/benthos/transformers/transform_first_name.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -66,6 +67,17 @@ func init() { } } +func NewTransformFirstNameOptsFromConfig(config *mgmtv1alpha1.TransformFirstName, maxLength *int64) (*TransformFirstNameOpts, error) { + if config == nil { + return NewTransformFirstNameOpts(nil, nil, nil) + } + return NewTransformFirstNameOpts( + maxLength, + config.PreserveLength, + nil, + ) +} + func (t *TransformFirstName) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformFirstNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_float.go b/worker/pkg/benthos/transformers/transform_float.go index 703cb3357c..9730112d28 100644 --- a/worker/pkg/benthos/transformers/transform_float.go +++ b/worker/pkg/benthos/transformers/transform_float.go @@ -6,6 +6,7 @@ import ( "strconv" "sync" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -74,6 +75,19 @@ func init() { } } +func NewTransformFloat64OptsFromConfig(config *mgmtv1alpha1.TransformFloat64, scale, precision *int64) (*TransformFloat64Opts, error) { + if config == nil { + return NewTransformFloat64Opts(nil, nil, nil, nil, nil) + } + return NewTransformFloat64Opts( + config.RandomizationRangeMin, + config.RandomizationRangeMax, + precision, + scale, + nil, + ) +} + func (t *TransformFloat64) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformFloat64Opts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_full_name.go b/worker/pkg/benthos/transformers/transform_full_name.go index 6c529127e9..465e934559 100644 --- a/worker/pkg/benthos/transformers/transform_full_name.go +++ b/worker/pkg/benthos/transformers/transform_full_name.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -67,6 +68,17 @@ func init() { } } +func NewTransformFullNameOptsFromConfig(config *mgmtv1alpha1.TransformFullName, maxLength *int64) (*TransformFullNameOpts, error) { + if config == nil { + return NewTransformFullNameOpts(nil, nil, nil) + } + return NewTransformFullNameOpts( + maxLength, + config.PreserveLength, + nil, + ) +} + func (t *TransformFullName) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformFullNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_int64.go b/worker/pkg/benthos/transformers/transform_int64.go index ed3dcdbf82..24d861d6ca 100644 --- a/worker/pkg/benthos/transformers/transform_int64.go +++ b/worker/pkg/benthos/transformers/transform_int64.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -61,6 +62,17 @@ func init() { } } +func NewTransformInt64OptsFromConfig(config *mgmtv1alpha1.TransformInt64) (*TransformInt64Opts, error) { + if config == nil { + return NewTransformInt64Opts(nil, nil, nil) + } + return NewTransformInt64Opts( + config.RandomizationRangeMin, + config.RandomizationRangeMax, + nil, + ) +} + func (t *TransformInt64) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformInt64Opts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_int64_phone_number.go b/worker/pkg/benthos/transformers/transform_int64_phone_number.go index 3897fb250c..00a8082bab 100644 --- a/worker/pkg/benthos/transformers/transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_int64_phone_number.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformers_dataset "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/data-sets" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -61,6 +62,13 @@ func init() { } } +func NewTransformInt64PhoneNumberOptsFromConfig(config *mgmtv1alpha1.TransformInt64PhoneNumber) (*TransformInt64PhoneNumberOpts, error) { + if config == nil { + return NewTransformInt64PhoneNumberOpts(nil, nil) + } + return NewTransformInt64PhoneNumberOpts(config.PreserveLength, nil) +} + func (t *TransformInt64PhoneNumber) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformInt64PhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_lastname.go b/worker/pkg/benthos/transformers/transform_lastname.go index 105ee56a6d..04f71abc9b 100644 --- a/worker/pkg/benthos/transformers/transform_lastname.go +++ b/worker/pkg/benthos/transformers/transform_lastname.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -65,6 +66,17 @@ func init() { } } +func NewTransformLastNameOptsFromConfig(config *mgmtv1alpha1.TransformLastName, maxLength *int64) (*TransformLastNameOpts, error) { + if config == nil { + return NewTransformLastNameOpts(nil, nil, nil) + } + return NewTransformLastNameOpts( + maxLength, + config.PreserveLength, + nil, + ) +} + func (t *TransformLastName) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformLastNameOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_string.go b/worker/pkg/benthos/transformers/transform_string.go index 41234fdd4c..f18e246c25 100644 --- a/worker/pkg/benthos/transformers/transform_string.go +++ b/worker/pkg/benthos/transformers/transform_string.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -67,6 +68,18 @@ func init() { } } +func NewTransformStringOptsFromConfig(config *mgmtv1alpha1.TransformString, minLength, maxLength *int64) (*TransformStringOpts, error) { + if config == nil { + return NewTransformStringOpts(nil, nil, nil, nil) + } + return NewTransformStringOpts( + config.PreserveLength, + minLength, + maxLength, + nil, + ) +} + func (t *TransformString) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformStringOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transform_string_phone_number.go b/worker/pkg/benthos/transformers/transform_string_phone_number.go index e587089f12..fce89e77ca 100644 --- a/worker/pkg/benthos/transformers/transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/transform_string_phone_number.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" "github.com/warpstreamlabs/bento/public/bloblang" @@ -61,6 +62,16 @@ func init() { } } +func NewTransformStringPhoneNumberOptsFromConfig(config *mgmtv1alpha1.TransformPhoneNumber, maxLength *int64) (*TransformStringPhoneNumberOpts, error) { + if config == nil { + return NewTransformStringPhoneNumberOpts(nil, nil, nil) + } + return NewTransformStringPhoneNumberOpts( + config.PreserveLength, + maxLength, nil, + ) +} + func (t *TransformStringPhoneNumber) Transform(value, opts any) (any, error) { parsedOpts, ok := opts.(*TransformStringPhoneNumberOpts) if !ok { diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index f6934553cb..28b42345c6 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -56,11 +56,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig: config := transformerConfig.GetGenerateCategoricalConfig() - // var categories *string - // if config != nil && config.GetCategories() != "" { - // categories = &config.Categories - // } - opts, err := NewGenerateCategoricalOpts(config.Categories, nil) + opts, err := NewGenerateCategoricalOptsFromConfig(config) if err != nil { return nil, err } @@ -73,7 +69,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateBoolConfig: - opts, err := NewGenerateBoolOpts(nil) + config := transformerConfig.GetGenerateBoolConfig() + opts, err := NewGenerateBoolOptsFromConfig(config) if err != nil { return nil, err } @@ -86,9 +83,9 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformStringConfig: - pl := transformerConfig.GetTransformStringConfig().GetPreserveLength() + config := transformerConfig.GetTransformStringConfig() minLength := int64(3) // TODO: pull this value from the database schema - opts, err := NewTransformStringOpts(&pl, &minLength, &maxLength, nil) + opts, err := NewTransformStringOptsFromConfig(config, &minLength, &maxLength) if err != nil { return nil, err } @@ -101,7 +98,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformInt64Config: config := transformerConfig.GetTransformInt64Config() - opts, err := NewTransformInt64Opts(config.RandomizationRangeMin, config.RandomizationRangeMax, nil) + opts, err := NewTransformInt64OptsFromConfig(config) if err != nil { return nil, err } @@ -114,8 +111,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformFullNameConfig: - pl := transformerConfig.GetTransformFullNameConfig().GetPreserveLength() - opts, err := NewTransformFullNameOpts(&maxLength, &pl, nil) + config := transformerConfig.GetTransformFullNameConfig() + opts, err := NewTransformFullNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -129,12 +126,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateEmailConfig: config := transformerConfig.GetGenerateEmailConfig() - var emailType *string - if config.EmailType != nil { - emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() - emailType = &emailTypeStr - } - opts, err := NewGenerateEmailOpts(&maxLength, emailType, nil) + opts, err := NewGenerateEmailOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -148,26 +140,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformEmailConfig: config := transformerConfig.GetTransformEmailConfig() - var emailType *string - if config.EmailType != nil { - emailTypeStr := dtoEmailTypeToTransformerEmailType(config.GetEmailType()).String() - emailType = &emailTypeStr - } - var invalidEmailAction *string - if config.InvalidEmailAction != nil { - invalidEmailActionStr := dtoInvalidEmailActionToTransformerInvalidEmailAction(config.GetInvalidEmailAction()).String() - invalidEmailAction = &invalidEmailActionStr - } - var excludedDomains any = config.GetExcludedDomains() - opts, err := NewTransformEmailOpts( - config.PreserveDomain, - config.PreserveLength, - &excludedDomains, - &maxLength, - nil, - emailType, - invalidEmailAction, - ) + opts, err := NewTransformEmailOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -181,7 +154,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig: config := transformerConfig.GetGenerateCardNumberConfig() - opts, err := NewGenerateCardNumberOpts(config.ValidLuhn, nil) + opts, err := NewGenerateCardNumberOptsFromConfig(config) if err != nil { return nil, err } @@ -194,7 +167,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCityConfig: - opts, err := NewGenerateCityOpts(&maxLength, nil) + config := transformerConfig.GetGenerateCityConfig() + opts, err := NewGenerateCityOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -208,12 +182,10 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: config := transformerConfig.GetGenerateE164PhoneNumberConfig() - opts, err := NewGenerateInternationalPhoneNumberOpts(config.Min, config.Max, nil) + opts, err := NewGenerateInternationalPhoneNumberOptsFromConfig(config) if err != nil { return nil, err } - fmt.Println(opts.min) - fmt.Println(opts.max) generate := NewGenerateInternationalPhoneNumber().Generate return &TransformerExecutor{ Opts: opts, @@ -222,7 +194,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, }, nil case *mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig: - opts, err := NewGenerateFirstNameOpts(&maxLength, nil) + config := transformerConfig.GetGenerateFirstNameConfig() + opts, err := NewGenerateFirstNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -236,18 +209,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateFloat64Config: config := transformerConfig.GetGenerateFloat64Config() - var precision *int64 - if config.GetPrecision() != 0 { - precision = config.Precision - } - opts, err := NewGenerateFloat64Opts( - config.RandomizeSign, - config.Min, - config.Max, - precision, - nil, // TODO: update scale based on colInfo if available - nil, - ) + opts, err := NewGenerateFloat64OptsFromConfig(config, nil) if err != nil { return nil, err } @@ -260,7 +222,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig: - opts, err := NewGenerateFullAddressOpts(&maxLength, nil) + config := transformerConfig.GetGenerateFullAddressConfig() + opts, err := NewGenerateFullAddressOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -273,7 +236,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateFullNameConfig: - opts, err := NewGenerateFullNameOpts(&maxLength, nil) + config := transformerConfig.GetGenerateFullNameConfig() + opts, err := NewGenerateFullNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -287,7 +251,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateGenderConfig: config := transformerConfig.GetGenerateGenderConfig() - opts, err := NewGenerateGenderOpts(config.Abbreviate, &maxLength, nil) + opts, err := NewGenerateGenderOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -300,7 +264,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateInt64PhoneNumberConfig: - opts, err := NewGenerateInt64PhoneNumberOpts(nil) + config := transformerConfig.GetGenerateInt64PhoneNumberConfig() + opts, err := NewGenerateInt64PhoneNumberOptsFromConfig(config) if err != nil { return nil, err } @@ -314,7 +279,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateInt64Config: config := transformerConfig.GetGenerateInt64Config() - opts, err := NewGenerateInt64Opts(config.RandomizeSign, config.Min, config.Max, nil) + opts, err := NewGenerateInt64OptsFromConfig(config) if err != nil { return nil, err } @@ -327,7 +292,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateLastNameConfig: - opts, err := NewGenerateLastNameOpts(&maxLength, nil) + config := transformerConfig.GetGenerateLastNameConfig() + opts, err := NewGenerateLastNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -340,7 +306,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateSha256HashConfig: - opts, err := NewGenerateSHA256HashOpts() + config := transformerConfig.GetGenerateSha256HashConfig() + opts, err := NewGenerateSHA256HashOptsFromConfig(config) if err != nil { return nil, err } @@ -353,7 +320,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateSsnConfig: - opts, err := NewGenerateSSNOpts(nil) + config := transformerConfig.GetGenerateSsnConfig() + opts, err := NewGenerateSSNOptsFromConfig(config) if err != nil { return nil, err } @@ -366,8 +334,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateStateConfig: - generateFullName := transformerConfig.GetGenerateStateConfig().GetGenerateFullName() - opts, err := NewGenerateStateOpts(&generateFullName, nil) + config := transformerConfig.GetGenerateStateConfig() + opts, err := NewGenerateStateOptsFromConfig(config) if err != nil { return nil, err } @@ -380,7 +348,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig: - opts, err := NewGenerateStreetAddressOpts(&maxLength, nil) + config := transformerConfig.GetGenerateStreetAddressConfig() + opts, err := NewGenerateStreetAddressOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -394,7 +363,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig: config := transformerConfig.GetGenerateStringPhoneNumberConfig() - opts, err := NewGenerateStringPhoneNumberOpts(config.Min, config.Max, nil) + opts, err := NewGenerateStringPhoneNumberOptsFromConfig(config) if err != nil { return nil, err } @@ -408,7 +377,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOpts(config.Min, config.Max, nil) + opts, err := NewGenerateRandomStringOptsFromConfig(config) if err != nil { return nil, err } @@ -421,7 +390,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateUnixtimestampConfig: - opts, err := NewGenerateUnixTimestampOpts(nil) + config := transformerConfig.GetGenerateUnixtimestampConfig() + opts, err := NewGenerateUnixTimestampOptsFromConfig(config) if err != nil { return nil, err } @@ -434,7 +404,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateUsernameConfig: - opts, err := NewGenerateUsernameOpts(&maxLength, nil) + config := transformerConfig.GetGenerateUsernameConfig() + opts, err := NewGenerateUsernameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -447,7 +418,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateUtctimestampConfig: - opts, err := NewGenerateUTCTimestampOpts(nil) + config := transformerConfig.GetGenerateUtctimestampConfig() + opts, err := NewGenerateUTCTimestampOptsFromConfig(config) if err != nil { return nil, err } @@ -460,8 +432,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateUuidConfig: - ih := transformerConfig.GetGenerateUuidConfig().GetIncludeHyphens() - opts, err := NewGenerateUUIDOpts(&ih) + config := transformerConfig.GetGenerateUuidConfig() + opts, err := NewGenerateUUIDOptsFromConfig(config) if err != nil { return nil, err } @@ -474,7 +446,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateZipcodeConfig: - opts, err := NewGenerateZipcodeOpts(nil) + config := transformerConfig.GetGenerateZipcodeConfig() + opts, err := NewGenerateZipcodeOptsFromConfig(config) if err != nil { return nil, err } @@ -488,7 +461,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig: config := transformerConfig.GetTransformE164PhoneNumberConfig() - opts, err := NewTransformE164PhoneNumberOpts(config.PreserveLength, nil, nil) + opts, err := NewTransformE164PhoneNumberOptsFromConfig(config, nil) if err != nil { return nil, err } @@ -502,7 +475,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig: config := transformerConfig.GetTransformFirstNameConfig() - opts, err := NewTransformFirstNameOpts(&maxLength, config.PreserveLength, nil) + opts, err := NewTransformFirstNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -516,13 +489,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFloat64Config: config := transformerConfig.GetTransformFloat64Config() - opts, err := NewTransformFloat64Opts( - config.RandomizationRangeMin, - config.RandomizationRangeMax, - nil, // TODO: update precision based on colInfo if available - nil, // TODO: update scale based on colInfo if available - nil, - ) + opts, err := NewTransformFloat64OptsFromConfig(config, nil, nil) if err != nil { return nil, err } @@ -536,7 +503,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig: config := transformerConfig.GetTransformInt64PhoneNumberConfig() - opts, err := NewTransformInt64PhoneNumberOpts(config.PreserveLength, nil) + opts, err := NewTransformInt64PhoneNumberOptsFromConfig(config) if err != nil { return nil, err } @@ -550,7 +517,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformLastNameConfig: config := transformerConfig.GetTransformLastNameConfig() - opts, err := NewTransformLastNameOpts(&maxLength, config.PreserveLength, nil) + opts, err := NewTransformLastNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -564,7 +531,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformStringPhoneNumberOpts(config.PreserveLength, &maxLength, nil) + opts, err := NewTransformStringPhoneNumberOptsFromConfig(config, &maxLength) if err != nil { return nil, err } @@ -594,7 +561,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformCharacterScrambleConfig: config := transformerConfig.GetTransformCharacterScrambleConfig() - opts, err := NewTransformCharacterScrambleOpts(config.UserProvidedRegex, nil) + opts, err := NewTransformCharacterScrambleOptsFromConfig(config) if err != nil { return nil, err } @@ -607,8 +574,8 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCountryConfig: - generateFullName := transformerConfig.GetGenerateCountryConfig().GenerateFullName - opts, err := NewGenerateCountryOpts(generateFullName, nil) + config := transformerConfig.GetGenerateCountryConfig() + opts, err := NewGenerateCountryOptsFromConfig(config) if err != nil { return nil, err } @@ -645,25 +612,3 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform return nil, fmt.Errorf("unsupported transformer: %v", transformerConfig) } } - -func dtoEmailTypeToTransformerEmailType(dto mgmtv1alpha1.GenerateEmailType) GenerateEmailType { - switch dto { - case mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME: - return GenerateEmailType_FullName - default: - return GenerateEmailType_UuidV4 - } -} - -func dtoInvalidEmailActionToTransformerInvalidEmailAction(dto mgmtv1alpha1.InvalidEmailAction) InvalidEmailAction { - switch dto { - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE: - return InvalidEmailAction_Generate - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_NULL: - return InvalidEmailAction_Null - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_PASSTHROUGH: - return InvalidEmailAction_Passthrough - default: - return InvalidEmailAction_Reject - } -} diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformers/transformer_initializer_test.go index 88281b40f4..e93c891804 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer_test.go +++ b/worker/pkg/benthos/transformers/transformer_initializer_test.go @@ -55,7 +55,7 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) - t.Run("GenerateCategoricalConfig_NilConfig", func(t *testing.T) { + t.Run("GenerateCategoricalConfig_Nil", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{}, } @@ -79,6 +79,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.IsType(t, bool(true), result) }) + t.Run("GenerateBoolConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateBoolConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, bool(true), result) + }) + t.Run("TransformStringConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -110,6 +122,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("TransformStringConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test", executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("TransformInt64Config", func(t *testing.T) { rmin, rmax := int64(1), int64(5) config := &mgmtv1alpha1.TransformerConfig{ @@ -143,6 +167,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("TransformInt64Config_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(int64(50), executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("TransformFullNameConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -173,6 +209,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("TransformFullNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("John Doe", executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("GenerateEmailConfig", func(t *testing.T) { emailType := mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME config := &mgmtv1alpha1.TransformerConfig{ @@ -204,6 +252,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateEmailConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateEmailConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) + t.Run("TransformEmailConfig", func(t *testing.T) { preserve := true emailType := mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME @@ -240,6 +300,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, result) }) + t.Run("TransformEmailConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ + TransformEmailConfig: &mgmtv1alpha1.TransformEmail{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("test@example.com", executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + }) t.Run("GenerateCardNumberConfig", func(t *testing.T) { valid := true @@ -272,6 +345,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^\d{13,19}$`, result) }) + t.Run("GenerateCardNumber_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ + GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.Regexp(t, `^\d{13,19}$`, result) + }) + t.Run("GenerateCityConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCityConfig{ @@ -287,6 +374,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateCityConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCityConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateInternationalPhoneNumberConfig", func(t *testing.T) { rmin, rmax := int64(10), int64(10) config := &mgmtv1alpha1.TransformerConfig{ @@ -346,6 +446,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateFirstNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateFloat64Config", func(t *testing.T) { randomizeSign := true rmin, rmax := float64(-10), float64(10) @@ -386,6 +499,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateFloat64Config_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, float64(0), result) + require.NotEmpty(t, result) + }) + t.Run("GenerateFullAddressConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig{ @@ -401,6 +527,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateFullAddressConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateFullNameConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFullNameConfig{ @@ -416,6 +555,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateFullNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateFullNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateGenderConfig", func(t *testing.T) { abb := true config := &mgmtv1alpha1.TransformerConfig{ @@ -450,6 +602,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateGenderConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateInt64PhoneNumberConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64PhoneNumberConfig{ @@ -465,6 +630,21 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^[1-9]\d{9}$`, strconv.FormatInt(result.(int64), 10)) }) + t.Run("GenerateInt64PhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64PhoneNumberConfig{ + GenerateInt64PhoneNumberConfig: &mgmtv1alpha1.GenerateInt64PhoneNumber{}, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.Regexp(t, `^[1-9]\d{9}$`, strconv.FormatInt(result.(int64), 10)) + }) + t.Run("GenerateInt64Config", func(t *testing.T) { randomizeSign := true rmin, rmax := int64(-1000), int64(1000) @@ -503,6 +683,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateInt64Config_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.NotEmpty(t, result) + }) + t.Run("GenerateLastNameConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateLastNameConfig{ @@ -519,6 +712,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^[A-Z][a-z]+$`, result) }) + t.Run("GenerateLastNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateLastNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Regexp(t, `^[A-Z][a-z]+$`, result) + }) + t.Run("GenerateSha256HashConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateSha256HashConfig{ @@ -534,6 +741,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^[a-f0-9]{64}$`, result) }) + t.Run("GenerateSha256HashConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateSha256HashConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.Regexp(t, `^[a-f0-9]{64}$`, result) + }) + t.Run("GenerateSsnConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateSsnConfig{ @@ -549,6 +769,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Regexp(t, `^\d{3}-\d{2}-\d{4}$`, result) }) + t.Run("GenerateSsnConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateSsnConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.Regexp(t, `^\d{3}-\d{2}-\d{4}$`, result) + }) + t.Run("GenerateStateConfig", func(t *testing.T) { genFullName := true config := &mgmtv1alpha1.TransformerConfig{ @@ -584,6 +817,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Len(t, result, 2) }) + t.Run("GenerateStateConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.Len(t, result, 2) + }) + t.Run("GenerateStreetAddressConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig{ @@ -599,6 +846,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateStreetAddressConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateStringPhoneNumberConfig", func(t *testing.T) { rmin, rmax := int64(10), int64(12) config := &mgmtv1alpha1.TransformerConfig{ @@ -633,6 +893,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateStringPhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateStringConfig", func(t *testing.T) { rmin, rmax := int64(5), int64(10) config := &mgmtv1alpha1.TransformerConfig{ @@ -668,6 +941,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateStringConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateUnixtimestampConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUnixtimestampConfig{ @@ -683,6 +969,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateUnixtimestampConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUnixtimestampConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, int64(0), result) + require.NotEmpty(t, result) + }) + t.Run("GenerateUsernameConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUsernameConfig{ @@ -698,6 +997,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateUsernameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUsernameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateUtctimestampConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUtctimestampConfig{ @@ -712,6 +1024,18 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.IsType(t, time.Now(), result) }) + t.Run("GenerateUtctimestampConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUtctimestampConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, time.Now(), result) + }) + t.Run("GenerateUuidConfig", func(t *testing.T) { hyphens := false config := &mgmtv1alpha1.TransformerConfig{ @@ -745,6 +1069,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateUuidConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("GenerateZipcodeConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateZipcodeConfig{ @@ -760,6 +1097,19 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("GenerateZipcodeConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateZipcodeConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + }) + t.Run("TransformE164PhoneNumberConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -795,6 +1145,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEqual(t, originalNumber, result) }) + t.Run("TransformE164PhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "+12345678901" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalNumber, result) + }) + t.Run("TransformFirstNameConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -830,6 +1194,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("TransformFirstNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "John" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalName, result) + require.NotEmpty(t, result) + }) + t.Run("TransformFloat64Config", func(t *testing.T) { randomizationRangeMin := float64(-10.0) randomizationRangeMax := float64(10.0) @@ -869,6 +1247,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEqual(t, originalValue, transformedValue) }) + t.Run("TransformFloat64Config_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalValue := float64(5.5) + result, err := executor.Mutate(originalValue, executor.Opts) + require.NoError(t, err) + transformedValue := *result.(*float64) + require.NotEqual(t, originalValue, transformedValue) + }) + t.Run("TransformInt64PhoneNumberConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -907,6 +1299,21 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, transformedNumber) }) + t.Run("TransformInt64PhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := int64(1234567890) + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + transformedNumber := *result.(*int64) + require.NotEqual(t, originalNumber, transformedNumber) + require.NotEmpty(t, transformedNumber) + }) + t.Run("TransformLastNameConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -942,6 +1349,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEqual(t, originalName, result) }) + t.Run("TransformLastNameConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalName := "Smith" + result, err := executor.Mutate(originalName, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalName, result) + }) + t.Run("TransformPhoneNumberConfig", func(t *testing.T) { preserveLength := true config := &mgmtv1alpha1.TransformerConfig{ @@ -977,6 +1398,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEqual(t, originalNumber, result) }) + t.Run("TransformPhoneNumberConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalNumber := "123-456-7890" + result, err := executor.Mutate(originalNumber, executor.Opts) + require.NoError(t, err) + require.NotEmpty(t, result) + require.NotEqual(t, originalNumber, result) + }) + t.Run("Nullconfig", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_Nullconfig{ @@ -1040,6 +1475,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NotEmpty(t, result) }) + t.Run("TransformCharacterScrambleConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformCharacterScrambleConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + originalString := "Hello123World" + result, err := executor.Mutate(originalString, executor.Opts) + require.NoError(t, err) + require.NotEqual(t, originalString, result) + require.NotEmpty(t, result) + }) + t.Run("GenerateCountryConfig", func(t *testing.T) { genFull := true config := &mgmtv1alpha1.TransformerConfig{ @@ -1075,6 +1524,20 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.LessOrEqual(t, len(result.(string)), 2) }) + t.Run("GenerateCountryConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{}, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEmpty(t, result) + require.LessOrEqual(t, len(result.(string)), 2) + }) + t.Run("TransformPiiTextConfig_Empty", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformPiiTextConfig{ @@ -1113,6 +1576,42 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.Equal(t, mockText, result) }) + t.Run("TransformPiiTextConfig_Nil", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformPiiTextConfig{}, + } + + mockanalyze := presidioapi.NewMockAnalyzeInterface(t) + mockanon := presidioapi.NewMockAnonymizeInterface(t) + + mockanalyze.On("PostAnalyzeWithResponse", mock.Anything, mock.Anything). + Return(&presidioapi.PostAnalyzeResponse{ + JSON200: &[]presidioapi.RecognizerResultWithAnaysisExplanation{ + {}, + }, + }, nil) + + mockText := "bar" + mockanon.On("PostAnonymizeWithResponse", mock.Anything, mock.Anything). + Return(&presidioapi.PostAnonymizeResponse{ + JSON200: &presidioapi.AnonymizeResponse{Text: &mockText}, + }, nil) + + execOpts := []TransformerExecutorOption{ + WithTransformPiiTextConfig(mockanalyze, mockanon), + } + executor, err := InitializeTransformerByConfigType(config, execOpts...) + require.NoError(t, err) + require.NotNil(t, executor) + + originalText := "Hello, John Doe!" + result, err := executor.Mutate(originalText, executor.Opts) + require.NoError(t, err) + require.IsType(t, "", result) + require.NotEqual(t, originalText, result) + require.Equal(t, mockText, result) + }) + t.Run("UnsupportedConfig", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: nil, From 5367ccdf516b57809f2cd2de7fad8eba21015b78 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 14:46:36 -0700 Subject: [PATCH 15/32] make transformer config args optional in model --- .../anonymization-service_integration_test.go | 9 ++- .../system_transformers.go | 65 +++++++-------- backend/sql/postgresql/models/transformers.go | 64 +++++++-------- internal/gotypeutil/parsers.go | 4 + .../json-anonymizer/json-anonymizer_test.go | 3 +- .../benthos-builder_test.go | 81 ++++++++++--------- .../gen-benthos-configs/processors.go | 8 +- .../init-statement-builder_test.go | 19 ++--- .../workflow/workflow_integration_test.go | 13 +-- 9 files changed, 138 insertions(+), 128 deletions(-) diff --git a/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go b/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go index b03aab0dac..3f0438bd62 100644 --- a/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go +++ b/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go @@ -7,6 +7,7 @@ import ( "connectrpc.com/connect" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + "github.com/nucleuscloud/neosync/internal/gotypeutil" "github.com/stretchr/testify/require" ) @@ -53,8 +54,8 @@ func (s *IntegrationTestSuite) Test_AnonymizeService_AnonymizeMany() { N: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ - Min: 18, - Max: 30, + Min: gotypeutil.ToPtr(int64(18)), + Max: gotypeutil.ToPtr(int64(30)), }, }, }, @@ -124,8 +125,8 @@ func (s *IntegrationTestSuite) Test_AnonymizeService_AnonymizeSingle() { N: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ - Min: 18, - Max: 30, + Min: gotypeutil.ToPtr(int64(18)), + Max: gotypeutil.ToPtr(int64(30)), }, }, }, diff --git a/backend/services/mgmt/v1alpha1/transformers-service/system_transformers.go b/backend/services/mgmt/v1alpha1/transformers-service/system_transformers.go index 43d69034d2..c9c07b9d5a 100644 --- a/backend/services/mgmt/v1alpha1/transformers-service/system_transformers.go +++ b/backend/services/mgmt/v1alpha1/transformers-service/system_transformers.go @@ -9,6 +9,7 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" nucleuserrors "github.com/nucleuscloud/neosync/backend/internal/errors" ee_transformers "github.com/nucleuscloud/neosync/internal/ee/transformers" + "github.com/nucleuscloud/neosync/internal/gotypeutil" "github.com/spf13/viper" ) @@ -44,8 +45,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: false, - PreserveLength: false, + PreserveDomain: gotypeutil.ToPtr(false), + PreserveLength: gotypeutil.ToPtr(false), ExcludedDomains: []string{}, EmailType: &defaultGenerateEmailType, InvalidEmailAction: &defaultInvalidEmailAction, @@ -76,7 +77,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{ - ValidLuhn: true, + ValidLuhn: gotypeutil.ToPtr(true), }, }, }, @@ -117,8 +118,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{ - Min: 9, - Max: 15, + Min: gotypeutil.ToPtr(int64(9)), + Max: gotypeutil.ToPtr(int64(15)), }, }, }, @@ -146,10 +147,10 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{ GenerateFloat64Config: &mgmtv1alpha1.GenerateFloat64{ - RandomizeSign: false, - Min: 1.00, - Max: 100.00, - Precision: 6, + RandomizeSign: gotypeutil.ToPtr(false), + Min: gotypeutil.ToPtr(1.00), + Max: gotypeutil.ToPtr(100.00), + Precision: gotypeutil.ToPtr(int64(6)), }, }, }, @@ -190,7 +191,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{ GenerateGenderConfig: &mgmtv1alpha1.GenerateGender{ - Abbreviate: false, + Abbreviate: gotypeutil.ToPtr(false), }, }, }, @@ -218,9 +219,9 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ - RandomizeSign: false, - Min: 1, - Max: 40, + RandomizeSign: gotypeutil.ToPtr(false), + Min: gotypeutil.ToPtr(int64(1)), + Max: gotypeutil.ToPtr(int64(40)), }, }, }, @@ -274,7 +275,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{ GenerateStateConfig: &mgmtv1alpha1.GenerateState{ - GenerateFullName: false, + GenerateFullName: gotypeutil.ToPtr(false), }, }, }, @@ -302,8 +303,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{ GenerateStringPhoneNumberConfig: &mgmtv1alpha1.GenerateStringPhoneNumber{ - Min: 9, - Max: 14, + Min: gotypeutil.ToPtr(int64(9)), + Max: gotypeutil.ToPtr(int64(14)), }, }, }, @@ -318,8 +319,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ GenerateStringConfig: &mgmtv1alpha1.GenerateString{ - Min: 2, - Max: 7, + Min: gotypeutil.ToPtr(int64(2)), + Max: gotypeutil.ToPtr(int64(7)), }, }, }, @@ -373,7 +374,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -401,7 +402,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{ TransformE164PhoneNumberConfig: &mgmtv1alpha1.TransformE164PhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -416,7 +417,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ TransformFirstNameConfig: &mgmtv1alpha1.TransformFirstName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -431,8 +432,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{ - RandomizationRangeMin: 20.00, - RandomizationRangeMax: 50.00, + RandomizationRangeMin: gotypeutil.ToPtr(20.00), + RandomizationRangeMax: gotypeutil.ToPtr(50.00), }, }, }, @@ -447,7 +448,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -462,7 +463,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{ TransformInt64PhoneNumberConfig: &mgmtv1alpha1.TransformInt64PhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -477,8 +478,8 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ TransformInt64Config: &mgmtv1alpha1.TransformInt64{ - RandomizationRangeMin: 20, - RandomizationRangeMax: 50, + RandomizationRangeMin: gotypeutil.ToPtr(int64(20)), + RandomizationRangeMax: gotypeutil.ToPtr(int64(50)), }, }, }, @@ -493,7 +494,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{ TransformLastNameConfig: &mgmtv1alpha1.TransformLastName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -508,7 +509,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{ TransformPhoneNumberConfig: &mgmtv1alpha1.TransformPhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -523,7 +524,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ TransformStringConfig: &mgmtv1alpha1.TransformString{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -577,7 +578,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{ GenerateCategoricalConfig: &mgmtv1alpha1.GenerateCategorical{ - Categories: "value1,value2", + Categories: gotypeutil.ToPtr("value1,value2"), }, }, }, @@ -620,7 +621,7 @@ var ( Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{ GenerateCountryConfig: &mgmtv1alpha1.GenerateCountry{ - GenerateFullName: false, + GenerateFullName: gotypeutil.ToPtr(false), }, }, }, diff --git a/backend/sql/postgresql/models/transformers.go b/backend/sql/postgresql/models/transformers.go index 87d5b067bb..7fe7ae1909 100644 --- a/backend/sql/postgresql/models/transformers.go +++ b/backend/sql/postgresql/models/transformers.go @@ -60,8 +60,8 @@ type GenerateEmailConfig struct { } type TransformEmailConfig struct { - PreserveLength bool `json:"preserveLength"` - PreserveDomain bool `json:"preserveDomain"` + PreserveLength *bool `json:"preserveLength,omitempty"` + PreserveDomain *bool `json:"preserveDomain,omitempty"` ExcludedDomains []string `json:"excludedDomains"` EmailType *int32 `json:"emailType,omitempty"` InvalidEmailAction *int32 `json:"invalidEmailAction,omitempty"` @@ -70,7 +70,7 @@ type TransformEmailConfig struct { type GenerateBoolConfig struct{} type GenerateCardNumberConfig struct { - ValidLuhn bool `json:"validLuhn"` + ValidLuhn *bool `json:"validLuhn,omitempty"` } type GenerateCityConfig struct{} @@ -78,16 +78,16 @@ type GenerateCityConfig struct{} type GenerateDefaultConfig struct{} type GenerateE164PhoneNumberConfig struct { - Min int64 `json:"min"` - Max int64 `json:"max"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` } type GenerateFirstNameConfig struct{} type GenerateFloat64Config struct { - RandomizeSign bool `json:"randomizeSign"` - Min float64 `json:"min"` - Max float64 `json:"max"` - Precision int64 `json:"precision"` + RandomizeSign *bool `json:"randomizeSign,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + Precision *int64 `json:"precision,omitempty"` } type GenerateFullAddressConfig struct{} @@ -95,15 +95,15 @@ type GenerateFullAddressConfig struct{} type GenerateFullNameConfig struct{} type GenerateGenderConfig struct { - Abbreviate bool `json:"abbreviate"` + Abbreviate *bool `json:"abbreviate,omitempty"` } type GenerateInt64PhoneNumberConfig struct{} type GenerateInt64Config struct { - RandomizeSign bool `json:"randomizeSign"` - Min int64 `json:"min"` - Max int64 `json:"max"` + RandomizeSign *bool `json:"randomizeSign,omitempty"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` } type GenerateLastNameConfig struct{} @@ -113,19 +113,19 @@ type GenerateSha256HashConfig struct{} type GenerateSsnConfig struct{} type GenerateStateConfig struct { - GenerateFullName bool `json:"generateFullName"` + GenerateFullName *bool `json:"generateFullName,omitempty"` } type GenerateStreetAddressConfig struct{} type GenerateStringPhoneNumberConfig struct { - Min int64 `json:"min"` - Max int64 `json:"max"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` } type GenerateStringConfig struct { - Min int64 `json:"min"` - Max int64 `json:"max"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` } type GenerateUnixTimestampConfig struct{} @@ -134,47 +134,47 @@ type GenerateUsernameConfig struct{} type GenerateUtcTimestampConfig struct{} type GenerateUuidConfig struct { - IncludeHyphens bool `json:"includeHyphens"` + IncludeHyphens *bool `json:"includeHyphens,omitempty"` } type GenerateZipcodeConfig struct{} type TransformE164PhoneNumberConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformFirstNameConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformFloat64Config struct { - RandomizationRangeMin float64 `json:"randomizationRangeMin"` - RandomizationRangeMax float64 `json:"randomizationRangeMax"` + RandomizationRangeMin *float64 `json:"randomizationRangeMin,omitempty"` + RandomizationRangeMax *float64 `json:"randomizationRangeMax,omitempty"` } type TransformFullNameConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformInt64PhoneNumberConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformInt64Config struct { - RandomizationRangeMin int64 `json:"randomizationRangeMin"` - RandomizationRangeMax int64 `json:"randomizationRangeMax"` + RandomizationRangeMin *int64 `json:"randomizationRangeMin,omitempty"` + RandomizationRangeMax *int64 `json:"randomizationRangeMax,omitempty"` } type TransformLastNameConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformPhoneNumberConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type TransformStringConfig struct { - PreserveLength bool `json:"preserveLength"` + PreserveLength *bool `json:"preserveLength,omitempty"` } type PassthroughConfig struct{} @@ -190,7 +190,7 @@ type TransformJavascriptConfig struct { } type GenerateCategoricalConfig struct { - Categories string `json:"categories"` + Categories *string `json:"categories,omitempty"` } type TransformCharacterScramble struct { @@ -202,7 +202,7 @@ type GenerateJavascript struct { } type GenerateCountryConfig struct { - GenerateFullName bool `json:"generateFullName"` + GenerateFullName *bool `json:"generateFullName,omitempty"` } // from API -> DB diff --git a/internal/gotypeutil/parsers.go b/internal/gotypeutil/parsers.go index 63820a1f1e..b8621def03 100644 --- a/internal/gotypeutil/parsers.go +++ b/internal/gotypeutil/parsers.go @@ -50,3 +50,7 @@ func ParseSlice(input any) ([]any, error) { } return result, nil } + +func ToPtr[T any](value T) *T { + return &value +} diff --git a/internal/json-anonymizer/json-anonymizer_test.go b/internal/json-anonymizer/json-anonymizer_test.go index 800080a365..a8f02e2900 100644 --- a/internal/json-anonymizer/json-anonymizer_test.go +++ b/internal/json-anonymizer/json-anonymizer_test.go @@ -287,13 +287,14 @@ func Test_AnonymizeJSON_Largedata(t *testing.T) { inputStrings, inputObjects, err := getTestData("./testdata/company.json") require.NoError(t, err) + preserveLength := true // Define transformer mappings mappings := []*mgmtv1alpha1.TransformerMapping{ { Expression: ".companyName", Transformer: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ - TransformStringConfig: &mgmtv1alpha1.TransformString{PreserveLength: true}, + TransformStringConfig: &mgmtv1alpha1.TransformString{PreserveLength: &preserveLength}, }, }, }, diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go index 60e138df73..bd72dbf844 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go @@ -16,6 +16,7 @@ import ( sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" tabledependency "github.com/nucleuscloud/neosync/backend/pkg/table-dependency" pg_models "github.com/nucleuscloud/neosync/backend/sql/postgresql/models" + "github.com/nucleuscloud/neosync/internal/gotypeutil" "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/shared" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -371,8 +372,8 @@ func Test_buildProcessorConfigsMutation(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: true, - PreserveLength: false, + PreserveDomain: gotypeutil.ToPtr(true), + PreserveLength: gotypeutil.ToPtr(false), ExcludedDomains: []string{}, }, }, @@ -475,8 +476,8 @@ func Test_convertUserDefinedFunctionConfig(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: true, - PreserveLength: false, + PreserveDomain: gotypeutil.ToPtr(true), + PreserveLength: gotypeutil.ToPtr(false), ExcludedDomains: []string{}, }, }, @@ -500,8 +501,8 @@ func Test_convertUserDefinedFunctionConfig(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: true, - PreserveLength: false, + PreserveDomain: gotypeutil.ToPtr(true), + PreserveLength: gotypeutil.ToPtr(false), ExcludedDomains: []string{}, }, }, @@ -635,8 +636,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformEmailConfig{ TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ - PreserveDomain: false, - PreserveLength: false, + PreserveDomain: gotypeutil.ToPtr(false), + PreserveLength: gotypeutil.ToPtr(false), ExcludedDomains: []string{}, EmailType: &uuidEmailType, }, @@ -656,7 +657,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ GenerateCardNumberConfig: &mgmtv1alpha1.GenerateCardNumber{ - ValidLuhn: true, + ValidLuhn: gotypeutil.ToPtr(true), }, }, }, @@ -674,8 +675,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig{ GenerateE164PhoneNumberConfig: &mgmtv1alpha1.GenerateE164PhoneNumber{ - Min: 9, - Max: 15, + Min: gotypeutil.ToPtr(int64(9)), + Max: gotypeutil.ToPtr(int64(15)), }, }, }, @@ -693,10 +694,10 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFloat64Config{ GenerateFloat64Config: &mgmtv1alpha1.GenerateFloat64{ - RandomizeSign: true, - Min: 1.00, - Max: 100.00, - Precision: 6, + RandomizeSign: gotypeutil.ToPtr(true), + Min: gotypeutil.ToPtr(1.00), + Max: gotypeutil.ToPtr(100.00), + Precision: gotypeutil.ToPtr(int64(6)), }, }, }, @@ -722,7 +723,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateGenderConfig{ GenerateGenderConfig: &mgmtv1alpha1.GenerateGender{ - Abbreviate: false, + Abbreviate: gotypeutil.ToPtr(false), }, }, }, @@ -740,9 +741,9 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{ GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{ - RandomizeSign: true, - Min: 1, - Max: 40, + RandomizeSign: gotypeutil.ToPtr(true), + Min: gotypeutil.ToPtr(int64(1)), + Max: gotypeutil.ToPtr(int64(40)), }, }, }, @@ -776,7 +777,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStateConfig{ GenerateStateConfig: &mgmtv1alpha1.GenerateState{ - GenerateFullName: false, + GenerateFullName: gotypeutil.ToPtr(false), }, }, }, @@ -794,8 +795,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig{ GenerateStringPhoneNumberConfig: &mgmtv1alpha1.GenerateStringPhoneNumber{ - Min: 9, - Max: 14, + Min: gotypeutil.ToPtr(int64(9)), + Max: gotypeutil.ToPtr(int64(14)), }, }, }, @@ -805,8 +806,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ GenerateStringConfig: &mgmtv1alpha1.GenerateString{ - Min: 2, - Max: 7, + Min: gotypeutil.ToPtr(int64(2)), + Max: gotypeutil.ToPtr(int64(7)), }, }, }, @@ -840,7 +841,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -858,7 +859,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig{ TransformE164PhoneNumberConfig: &mgmtv1alpha1.TransformE164PhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -868,7 +869,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ TransformFirstNameConfig: &mgmtv1alpha1.TransformFirstName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -878,8 +879,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{ - RandomizationRangeMin: 20.00, - RandomizationRangeMax: 50.00, + RandomizationRangeMin: gotypeutil.ToPtr(20.00), + RandomizationRangeMax: gotypeutil.ToPtr(50.00), }, }, }, @@ -889,7 +890,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFullNameConfig{ TransformFullNameConfig: &mgmtv1alpha1.TransformFullName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -899,7 +900,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig{ TransformInt64PhoneNumberConfig: &mgmtv1alpha1.TransformInt64PhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -909,8 +910,8 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ TransformInt64Config: &mgmtv1alpha1.TransformInt64{ - RandomizationRangeMin: 20, - RandomizationRangeMax: 50, + RandomizationRangeMin: gotypeutil.ToPtr(int64(20)), + RandomizationRangeMax: gotypeutil.ToPtr(int64(50)), }, }, }, @@ -920,7 +921,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformLastNameConfig{ TransformLastNameConfig: &mgmtv1alpha1.TransformLastName{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -930,7 +931,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig{ TransformPhoneNumberConfig: &mgmtv1alpha1.TransformPhoneNumber{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -940,7 +941,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ TransformStringConfig: &mgmtv1alpha1.TransformString{ - PreserveLength: false, + PreserveLength: gotypeutil.ToPtr(false), }, }, }, @@ -950,7 +951,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig{ GenerateCategoricalConfig: &mgmtv1alpha1.GenerateCategorical{ - Categories: "value1,value2", + Categories: gotypeutil.ToPtr("value1,value2"), }, }, }, @@ -986,7 +987,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCountryConfig{ GenerateCountryConfig: &mgmtv1alpha1.GenerateCountry{ - GenerateFullName: false, + GenerateFullName: gotypeutil.ToPtr(false), }, }, }, @@ -1034,8 +1035,8 @@ func Test_computeMutationFunction_handles_Db_Maxlen(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateStringConfig{ GenerateStringConfig: &mgmtv1alpha1.GenerateString{ - Min: 2, - Max: 7, + Min: gotypeutil.ToPtr(int64(2)), + Max: gotypeutil.ToPtr(int64(7)), }, }, }, diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go index 8d602d0843..5750f364dc 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go @@ -539,14 +539,14 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STREET_ADDRESS: return fmt.Sprintf(`generate_street_address(max_length:%d)`, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STRING_PHONE_NUMBER: - minValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().Min - maxValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().Max + minValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().GetMin() + maxValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().GetMax() minValue = transformer_utils.MinInt(minValue, maxLen) maxValue = transformer_utils.Ceil(maxValue, maxLen) return fmt.Sprintf("generate_string_phone_number(min:%d,max:%d)", minValue, maxValue), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_RANDOM_STRING: - minValue := col.Transformer.Config.GetGenerateStringConfig().Min - maxValue := col.Transformer.Config.GetGenerateStringConfig().Max + minValue := col.Transformer.Config.GetGenerateStringConfig().GetMin() + maxValue := col.Transformer.Config.GetGenerateStringConfig().GetMax() minValue = transformer_utils.MinInt(minValue, maxLen) // ensure the min is not larger than the max allowed length maxValue = transformer_utils.Ceil(maxValue, maxLen) // todo: we need to pull in the min from the database schema diff --git a/worker/pkg/workflows/datasync/activities/run-sql-init-table-stmts/init-statement-builder_test.go b/worker/pkg/workflows/datasync/activities/run-sql-init-table-stmts/init-statement-builder_test.go index 924bdb1142..e73d480a5a 100644 --- a/worker/pkg/workflows/datasync/activities/run-sql-init-table-stmts/init-statement-builder_test.go +++ b/worker/pkg/workflows/datasync/activities/run-sql-init-table-stmts/init-statement-builder_test.go @@ -11,6 +11,7 @@ import ( "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" + "github.com/nucleuscloud/neosync/internal/gotypeutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -56,7 +57,7 @@ func Test_InitStatementBuilder_Pg_Generate_InitSchema(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -204,7 +205,7 @@ func Test_InitStatementBuilder_Pg_Generate_NoInitStatement(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -307,7 +308,7 @@ func Test_InitStatementBuilder_Pg_TruncateCascade(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -335,7 +336,7 @@ func Test_InitStatementBuilder_Pg_TruncateCascade(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -439,7 +440,7 @@ func Test_InitStatementBuilder_Pg_Truncate(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -467,7 +468,7 @@ func Test_InitStatementBuilder_Pg_Truncate(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -579,7 +580,7 @@ func Test_InitStatementBuilder_Pg_InitSchema(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -607,7 +608,7 @@ func Test_InitStatementBuilder_Pg_InitSchema(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, @@ -722,7 +723,7 @@ func Test_InitStatementBuilder_Mysql_Generate(t *testing.T) { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, diff --git a/worker/pkg/workflows/datasync/workflow/workflow_integration_test.go b/worker/pkg/workflows/datasync/workflow/workflow_integration_test.go index 3af8e4e0aa..77153757e5 100644 --- a/worker/pkg/workflows/datasync/workflow/workflow_integration_test.go +++ b/worker/pkg/workflows/datasync/workflow/workflow_integration_test.go @@ -19,6 +19,7 @@ import ( "github.com/nucleuscloud/neosync/backend/pkg/sqlconnect" sql_manager "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" + "github.com/nucleuscloud/neosync/internal/gotypeutil" accountstatus_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/account-status" genbenthosconfigs_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/gen-benthos-configs" runsqlinittablestmts_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/run-sql-init-table-stmts" @@ -1072,7 +1073,7 @@ func getAllDynamoDBSyncTests() map[string][]*workflow_testdata.IntegrationTest { Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64, Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ - TransformInt64Config: &mgmtv1alpha1.TransformInt64{RandomizationRangeMin: 10, RandomizationRangeMax: 1000}, + TransformInt64Config: &mgmtv1alpha1.TransformInt64{RandomizationRangeMin: gotypeutil.ToPtr(int64(10)), RandomizationRangeMax: gotypeutil.ToPtr(int64(1000))}, }, }, }, @@ -1309,7 +1310,7 @@ func getAllMongoDBSyncTests() map[string][]*workflow_testdata.IntegrationTest { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformStringConfig{ TransformStringConfig: &mgmtv1alpha1.TransformString{ - PreserveLength: true, + PreserveLength: gotypeutil.ToPtr(true), }, }, }, @@ -1337,8 +1338,8 @@ func getAllMongoDBSyncTests() map[string][]*workflow_testdata.IntegrationTest { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFloat64Config{ TransformFloat64Config: &mgmtv1alpha1.TransformFloat64{ - RandomizationRangeMin: 0, - RandomizationRangeMax: 300, + RandomizationRangeMin: gotypeutil.ToPtr(float64(0)), + RandomizationRangeMax: gotypeutil.ToPtr(float64(300)), }, }, }, @@ -1353,8 +1354,8 @@ func getAllMongoDBSyncTests() map[string][]*workflow_testdata.IntegrationTest { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformInt64Config{ TransformInt64Config: &mgmtv1alpha1.TransformInt64{ - RandomizationRangeMin: 0, - RandomizationRangeMax: 300, + RandomizationRangeMin: gotypeutil.ToPtr(int64(0)), + RandomizationRangeMax: gotypeutil.ToPtr(int64(300)), }, }, }, From 9cf030d8bb631257b06de6b294cd566a189a64bc Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 15:24:13 -0700 Subject: [PATCH 16/32] fix ptr types --- .../gen-benthos-configs/processors.go | 56 +++++++++---------- .../testdata/mysql/composite-keys/tests.go | 3 +- .../testdata/primary-key-transformer/tests.go | 3 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go index 5750f364dc..13b643759d 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go @@ -435,7 +435,7 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s switch col.Transformer.Source { case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CATEGORICAL: - categories := col.Transformer.Config.GetGenerateCategoricalConfig().Categories + categories := col.Transformer.Config.GetGenerateCategoricalConfig().GetCategories() return fmt.Sprintf(`generate_categorical(categories: %q)`, categories), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_EMAIL: emailType := col.GetTransformer().GetConfig().GetGenerateEmailConfig().GetEmailType() @@ -444,9 +444,9 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s } return fmt.Sprintf(`generate_email(max_length:%d,email_type:%q)`, maxLen, dtoEmailTypeToBenthosEmailType(emailType)), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_EMAIL: - pd := col.Transformer.Config.GetTransformEmailConfig().PreserveDomain - pl := col.Transformer.Config.GetTransformEmailConfig().PreserveLength - excludedDomains := col.Transformer.Config.GetTransformEmailConfig().ExcludedDomains + pd := col.Transformer.Config.GetTransformEmailConfig().GetPreserveDomain() + pl := col.Transformer.Config.GetTransformEmailConfig().GetPreserveLength() + excludedDomains := col.Transformer.Config.GetTransformEmailConfig().GetExcludedDomains() excludedDomainsStr, err := convertStringSliceToString(excludedDomains) if err != nil { @@ -469,20 +469,20 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_BOOL: return "generate_bool()", nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CARD_NUMBER: - luhn := col.Transformer.Config.GetGenerateCardNumberConfig().ValidLuhn + luhn := col.Transformer.Config.GetGenerateCardNumberConfig().GetValidLuhn() return fmt.Sprintf(`generate_card_number(valid_luhn:%t)`, luhn), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CITY: return fmt.Sprintf(`generate_city(max_length:%d)`, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_E164_PHONE_NUMBER: - minValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().Min - maxValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().Max + minValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().GetMin() + maxValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().GetMax() return fmt.Sprintf(`generate_e164_phone_number(min:%d,max:%d)`, minValue, maxValue), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FIRST_NAME: return fmt.Sprintf(`generate_first_name(max_length:%d)`, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FLOAT64: - randomSign := col.Transformer.Config.GetGenerateFloat64Config().RandomizeSign - minValue := col.Transformer.Config.GetGenerateFloat64Config().Min - maxValue := col.Transformer.Config.GetGenerateFloat64Config().Max + randomSign := col.Transformer.Config.GetGenerateFloat64Config().GetRandomizeSign() + minValue := col.Transformer.Config.GetGenerateFloat64Config().GetMin() + maxValue := col.Transformer.Config.GetGenerateFloat64Config().GetMax() var precision *int64 if col.GetTransformer().GetConfig().GetGenerateFloat64Config().GetPrecision() > 0 { @@ -518,14 +518,14 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FULL_NAME: return fmt.Sprintf(`generate_full_name(max_length:%d)`, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_GENDER: - ab := col.Transformer.Config.GetGenerateGenderConfig().Abbreviate + ab := col.Transformer.Config.GetGenerateGenderConfig().GetAbbreviate() return fmt.Sprintf(`generate_gender(abbreviate:%t,max_length:%d)`, ab, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64_PHONE_NUMBER: return "generate_int64_phone_number()", nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64: - sign := col.Transformer.Config.GetGenerateInt64Config().RandomizeSign - minValue := col.Transformer.Config.GetGenerateInt64Config().Min - maxValue := col.Transformer.Config.GetGenerateInt64Config().Max + sign := col.Transformer.Config.GetGenerateInt64Config().GetRandomizeSign() + minValue := col.Transformer.Config.GetGenerateInt64Config().GetMin() + maxValue := col.Transformer.Config.GetGenerateInt64Config().GetMax() return fmt.Sprintf(`generate_int64(randomize_sign:%t,min:%d, max:%d)`, sign, minValue, maxValue), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_LAST_NAME: return fmt.Sprintf(`generate_last_name(max_length:%d)`, maxLen), nil @@ -534,7 +534,7 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_SSN: return "generate_ssn()", nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STATE: - generateFullName := col.Transformer.Config.GetGenerateStateConfig().GenerateFullName + generateFullName := col.Transformer.Config.GetGenerateStateConfig().GetGenerateFullName() return fmt.Sprintf(`generate_state(generate_full_name:%t)`, generateFullName), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STREET_ADDRESS: return fmt.Sprintf(`generate_street_address(max_length:%d)`, maxLen), nil @@ -558,19 +558,19 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UTCTIMESTAMP: return "generate_utctimestamp()", nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UUID: - ih := col.Transformer.Config.GetGenerateUuidConfig().IncludeHyphens + ih := col.Transformer.Config.GetGenerateUuidConfig().GetIncludeHyphens() return fmt.Sprintf("generate_uuid(include_hyphens:%t)", ih), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_ZIPCODE: return "generate_zipcode()", nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_E164_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformE164PhoneNumberConfig().PreserveLength + pl := col.Transformer.Config.GetTransformE164PhoneNumberConfig().GetPreserveLength() return fmt.Sprintf("transform_e164_phone_number(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FIRST_NAME: - pl := col.Transformer.Config.GetTransformFirstNameConfig().PreserveLength + pl := col.Transformer.Config.GetTransformFirstNameConfig().GetPreserveLength() return fmt.Sprintf("transform_first_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FLOAT64: - rMin := col.Transformer.Config.GetTransformFloat64Config().RandomizationRangeMin - rMax := col.Transformer.Config.GetTransformFloat64Config().RandomizationRangeMax + rMin := col.Transformer.Config.GetTransformFloat64Config().GetRandomizationRangeMin() + rMax := col.Transformer.Config.GetTransformFloat64Config().GetRandomizationRangeMax() var precision *int64 if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { @@ -598,23 +598,23 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s template := fmt.Sprintf(`transform_float64(%s)`, strings.Join(fnStr, ", ")) return fmt.Sprintf(template, params...), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FULL_NAME: - pl := col.Transformer.Config.GetTransformFullNameConfig().PreserveLength + pl := col.Transformer.Config.GetTransformFullNameConfig().GetPreserveLength() return fmt.Sprintf("transform_full_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformInt64PhoneNumberConfig().PreserveLength + pl := col.Transformer.Config.GetTransformInt64PhoneNumberConfig().GetPreserveLength() return fmt.Sprintf("transform_int64_phone_number(value:this.%s,preserve_length:%t)", formattedColPath, pl), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64: - rMin := col.Transformer.Config.GetTransformInt64Config().RandomizationRangeMin - rMax := col.Transformer.Config.GetTransformInt64Config().RandomizationRangeMax + rMin := col.Transformer.Config.GetTransformInt64Config().GetRandomizationRangeMin() + rMax := col.Transformer.Config.GetTransformInt64Config().GetRandomizationRangeMax() return fmt.Sprintf(`transform_int64(value:this.%s,randomization_range_min:%d,randomization_range_max:%d)`, formattedColPath, rMin, rMax), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_LAST_NAME: - pl := col.Transformer.Config.GetTransformLastNameConfig().PreserveLength + pl := col.Transformer.Config.GetTransformLastNameConfig().GetPreserveLength() return fmt.Sprintf("transform_last_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformPhoneNumberConfig().PreserveLength + pl := col.Transformer.Config.GetTransformPhoneNumberConfig().GetPreserveLength() return fmt.Sprintf("transform_phone_number(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_STRING: - pl := col.Transformer.Config.GetTransformStringConfig().PreserveLength + pl := col.Transformer.Config.GetTransformStringConfig().GetPreserveLength() minLength := int64(3) // todo: we need to pull in this value from the database schema return fmt.Sprintf(`transform_string(value:this.%s,preserve_length:%t,min_length:%d,max_length:%d)`, formattedColPath, pl, minLength, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_NULL: @@ -631,7 +631,7 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s return fmt.Sprintf(`transform_character_scramble(value:this.%s)`, formattedColPath), nil } case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_COUNTRY: - generateFullName := col.Transformer.Config.GetGenerateCountryConfig().GenerateFullName + generateFullName := col.Transformer.Config.GetGenerateCountryConfig().GetGenerateFullName() return fmt.Sprintf(`generate_country(generate_full_name:%t)`, generateFullName), nil default: diff --git a/worker/pkg/workflows/datasync/workflow/testdata/mysql/composite-keys/tests.go b/worker/pkg/workflows/datasync/workflow/testdata/mysql/composite-keys/tests.go index e7a8933dea..ba260078da 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/mysql/composite-keys/tests.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/mysql/composite-keys/tests.go @@ -2,6 +2,7 @@ package mysql_compositekeys import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + "github.com/nucleuscloud/neosync/internal/gotypeutil" workflow_testdata "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/workflow/testdata" ) @@ -43,7 +44,7 @@ func getPkTransformerJobmappings() []*mgmtv1alpha1.JobMapping { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, diff --git a/worker/pkg/workflows/datasync/workflow/testdata/primary-key-transformer/tests.go b/worker/pkg/workflows/datasync/workflow/testdata/primary-key-transformer/tests.go index 3a9e0d6362..098067bfd2 100644 --- a/worker/pkg/workflows/datasync/workflow/testdata/primary-key-transformer/tests.go +++ b/worker/pkg/workflows/datasync/workflow/testdata/primary-key-transformer/tests.go @@ -2,6 +2,7 @@ package testdata_primarykeytransformer import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + "github.com/nucleuscloud/neosync/internal/gotypeutil" workflow_testdata "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/workflow/testdata" ) @@ -42,7 +43,7 @@ func getPkTransformerJobmappings() []*mgmtv1alpha1.JobMapping { Config: &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateUuidConfig{ GenerateUuidConfig: &mgmtv1alpha1.GenerateUuid{ - IncludeHyphens: true, + IncludeHyphens: gotypeutil.ToPtr(true), }, }, }, From fd5779852f37653967dde1cffb2ed853d42d59c5 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 15:27:13 -0700 Subject: [PATCH 17/32] revert generator changes --- worker/pkg/benthos/transformers/gen_generate_bool.go | 2 +- worker/pkg/benthos/transformers/gen_generate_card_number.go | 2 +- worker/pkg/benthos/transformers/gen_generate_categorical.go | 2 +- worker/pkg/benthos/transformers/gen_generate_city.go | 2 +- worker/pkg/benthos/transformers/gen_generate_country.go | 2 +- worker/pkg/benthos/transformers/gen_generate_email.go | 2 +- worker/pkg/benthos/transformers/gen_generate_first_name.go | 2 +- worker/pkg/benthos/transformers/gen_generate_float.go | 2 +- .../pkg/benthos/transformers/gen_generate_full_address.go | 2 +- worker/pkg/benthos/transformers/gen_generate_full_name.go | 2 +- worker/pkg/benthos/transformers/gen_generate_gender.go | 2 +- worker/pkg/benthos/transformers/gen_generate_int64.go | 2 +- .../benthos/transformers/gen_generate_int64_phone_number.go | 2 +- .../transformers/gen_generate_international_phone_number.go | 2 +- worker/pkg/benthos/transformers/gen_generate_last_name.go | 2 +- .../pkg/benthos/transformers/gen_generate_random_string.go | 2 +- worker/pkg/benthos/transformers/gen_generate_ssn.go | 2 +- worker/pkg/benthos/transformers/gen_generate_state.go | 2 +- .../pkg/benthos/transformers/gen_generate_street_address.go | 2 +- .../transformers/gen_generate_string_phone_number.go | 2 +- .../pkg/benthos/transformers/gen_generate_unix_timestamp.go | 2 +- worker/pkg/benthos/transformers/gen_generate_username.go | 2 +- .../pkg/benthos/transformers/gen_generate_utc_timestamp.go | 2 +- worker/pkg/benthos/transformers/gen_generate_zipcode.go | 2 +- .../transformers/gen_transform_character_scramble.go | 2 +- .../benthos/transformers/gen_transform_e164_phone_number.go | 2 +- worker/pkg/benthos/transformers/gen_transform_email.go | 2 +- worker/pkg/benthos/transformers/gen_transform_first_name.go | 2 +- worker/pkg/benthos/transformers/gen_transform_float.go | 2 +- worker/pkg/benthos/transformers/gen_transform_full_name.go | 2 +- worker/pkg/benthos/transformers/gen_transform_int64.go | 2 +- .../transformers/gen_transform_int64_phone_number.go | 2 +- worker/pkg/benthos/transformers/gen_transform_lastname.go | 2 +- worker/pkg/benthos/transformers/gen_transform_string.go | 2 +- .../transformers/gen_transform_string_phone_number.go | 2 +- .../benthos/transformers/neosync_transformer_generator.go | 6 ++---- 36 files changed, 37 insertions(+), 39 deletions(-) diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index f01534234b..2bdc7d61c1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index 442343d5f4..98dc8ee0c9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index 5be1dfd5c9..582d0b0b88 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index 694256f7ab..e3adbed396 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index e14b9da9e9..4abd4caf9a 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index 17c24a8921..cf6debb37d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index 897b4ec67f..397f33289b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index e230e6c05e..d16fea18ad 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index c594d3db52..57725e83fb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index 3913f1ed25..329173951c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index cc2cfabdcf..c03848454d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index 2819cf0951..8c25f9ad37 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index 9bda577920..4b1cda478e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index 3227f39c06..b9934eddb4 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index 627aa17caf..0bf3ddea59 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index c7a51adf2f..517881c61d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index e8713dff7e..444c4dd79a 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index 3a6cbb432e..59d29160a1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index 1d96bb09a2..264fab4d9b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index 1086da46a6..a8fbcffca3 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index 6ebf9e5ace..f95717afe5 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index ac2ad8bf3a..6df1991fa0 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index 86713d2c51..cbd99cdb08 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index 4fb6c7705e..2e1ab3bfb9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 8422756295..5228484eb2 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index 1d3b709aef..e73f81a9bc 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index 1c9c09df11..1415f339bb 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index 29e17b7959..ec28e22383 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index e73d8a8425..39642acfd7 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index 9c188facb7..c6c2792733 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index 8384eafa64..c98491a801 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index d60ffb70cb..a51ed2a753 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index 4762049f33..f2e5020291 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index 8d7ad4b925..a773e07485 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 70b09bfe1e..65a0f6011e 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -7,8 +7,8 @@ package transformers import ( "fmt" - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" ) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index a73eb30060..8aa00921a2 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -100,8 +100,8 @@ import ( "fmt" {{ end }} {{- if eq .HasSeedParam true}} - "github.com/nucleuscloud/neosync/worker/pkg/rng" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" + "github.com/nucleuscloud/neosync/worker/pkg/rng" {{ end }} ) @@ -247,9 +247,7 @@ func generateCode(pkgName string, funcInfo *transformers.BenthosSpec) (string, e data.ImportFmt = true } } - t := template.Must(template.New("neosyncTransformerImpl").Funcs(template.FuncMap{ - "capitalizeFirst": capitalizeFirst, - }).Parse(codeTemplate)) + t := template.Must(template.New("neosyncTransformerImpl").Parse(codeTemplate)) var out bytes.Buffer err := t.Execute(&out, data) if err != nil { From d11359d7daaf8ac0ede18e1be300360f39439b36 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 15:29:40 -0700 Subject: [PATCH 18/32] remove unused code --- .../benthos/transformers/utils/type_utils.go | 19 ----- .../transformers/utils/type_utils_test.go | 77 ------------------- 2 files changed, 96 deletions(-) delete mode 100644 worker/pkg/benthos/transformers/utils/type_utils.go delete mode 100644 worker/pkg/benthos/transformers/utils/type_utils_test.go diff --git a/worker/pkg/benthos/transformers/utils/type_utils.go b/worker/pkg/benthos/transformers/utils/type_utils.go deleted file mode 100644 index 2b66194ce0..0000000000 --- a/worker/pkg/benthos/transformers/utils/type_utils.go +++ /dev/null @@ -1,19 +0,0 @@ -package transformer_utils - -import "reflect" - -func IsZeroValue[T any](value T) bool { - var zero T - - if reflect.DeepEqual(value, zero) { - return true - } - - v := reflect.ValueOf(value) - switch v.Kind() { - case reflect.Slice, reflect.Map: - return v.Len() == 0 - } - - return false -} diff --git a/worker/pkg/benthos/transformers/utils/type_utils_test.go b/worker/pkg/benthos/transformers/utils/type_utils_test.go deleted file mode 100644 index 99be36fdc1..0000000000 --- a/worker/pkg/benthos/transformers/utils/type_utils_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package transformer_utils - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_IsZeroValue(t *testing.T) { - t.Run("Primitive Types", func(t *testing.T) { - t.Run("Integer", func(t *testing.T) { - require.True(t, IsZeroValue(0)) - require.False(t, IsZeroValue(1)) - require.False(t, IsZeroValue(-1)) - }) - - t.Run("Float", func(t *testing.T) { - require.True(t, IsZeroValue(0.0)) - require.False(t, IsZeroValue(0.1)) - require.False(t, IsZeroValue(-0.1)) - }) - - t.Run("Boolean", func(t *testing.T) { - require.True(t, IsZeroValue(false)) - require.False(t, IsZeroValue(true)) - }) - - t.Run("String", func(t *testing.T) { - require.True(t, IsZeroValue("")) - require.False(t, IsZeroValue("hello")) - }) - }) - - t.Run("Complex Types", func(t *testing.T) { - t.Run("Slice", func(t *testing.T) { - require.True(t, IsZeroValue([]int(nil))) - require.True(t, IsZeroValue([]int{})) - require.False(t, IsZeroValue([]int{1, 2, 3})) - }) - - t.Run("Map", func(t *testing.T) { - require.True(t, IsZeroValue(map[string]int(nil))) - require.True(t, IsZeroValue(map[string]int{})) - require.False(t, IsZeroValue(map[string]int{"a": 1})) - }) - - t.Run("Struct", func(t *testing.T) { - type TestStruct struct { - A int - B string - } - require.True(t, IsZeroValue(TestStruct{})) - require.False(t, IsZeroValue(TestStruct{A: 1})) - require.False(t, IsZeroValue(TestStruct{B: "hello"})) - }) - - t.Run("Pointer", func(t *testing.T) { - var nilPtr *int - value := 5 - ptr := &value - require.True(t, IsZeroValue(nilPtr)) - require.False(t, IsZeroValue(ptr)) - }) - }) - - t.Run("Custom Types", func(t *testing.T) { - type CustomInt int - require.True(t, IsZeroValue(CustomInt(0))) - require.False(t, IsZeroValue(CustomInt(1))) - - type CustomStruct struct { - Value int - } - require.True(t, IsZeroValue(CustomStruct{})) - require.False(t, IsZeroValue(CustomStruct{Value: 1})) - }) -} From 4b8df8c2b95c51dc4b52724c6faefb341d785990 Mon Sep 17 00:00:00 2001 From: Alisha Date: Thu, 17 Oct 2024 15:36:13 -0700 Subject: [PATCH 19/32] fix spelling --- backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go | 2 +- backend/protos/mgmt/v1alpha1/transformer.proto | 2 +- docs/protos/data/proto_docs.json | 2 +- .../packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go index c2cc780d93..fdb34e7bf5 100644 --- a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go +++ b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go @@ -4566,7 +4566,7 @@ type PiiAnonymizer_Mask struct { MaskingChar *string `protobuf:"bytes,1,opt,name=masking_char,json=maskingChar,proto3,oneof" json:"masking_char,omitempty"` // How many characters to mask. CharsToMask *int32 `protobuf:"varint,2,opt,name=chars_to_mask,json=charsToMask,proto3,oneof" json:"chars_to_mask,omitempty"` - // Whether to mask the PII from start or enTransformEmaild + // Whether to mask the PII from start or end FromEnd *bool `protobuf:"varint,3,opt,name=from_end,json=fromEnd,proto3,oneof" json:"from_end,omitempty"` } diff --git a/backend/protos/mgmt/v1alpha1/transformer.proto b/backend/protos/mgmt/v1alpha1/transformer.proto index 719a7564bb..b3b44f9777 100644 --- a/backend/protos/mgmt/v1alpha1/transformer.proto +++ b/backend/protos/mgmt/v1alpha1/transformer.proto @@ -267,7 +267,7 @@ message PiiAnonymizer { optional string masking_char = 1; // How many characters to mask. optional int32 chars_to_mask = 2; - // Whether to mask the PII from start or enTransformEmaild + // Whether to mask the PII from start or end optional bool from_end = 3; } message Hash { diff --git a/docs/protos/data/proto_docs.json b/docs/protos/data/proto_docs.json index 062a3b1a81..287e766970 100644 --- a/docs/protos/data/proto_docs.json +++ b/docs/protos/data/proto_docs.json @@ -1512,7 +1512,7 @@ }, { "name": "from_end", - "description": "Whether to mask the PII from start or enTransformEmaild", + "description": "Whether to mask the PII from start or end", "label": "optional", "type": "bool", "longType": "bool", diff --git a/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts b/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts index 9a320fe6ad..ef97b8ec1d 100644 --- a/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts +++ b/frontend/packages/sdk/src/client/mgmt/v1alpha1/transformer_pb.ts @@ -1881,7 +1881,7 @@ export class PiiAnonymizer_Mask extends Message { charsToMask?: number; /** - * Whether to mask the PII from start or enTransformEmaild + * Whether to mask the PII from start or end * * @generated from field: optional bool from_end = 3; */ From 60bc95f0d76d53ec9ac2c533fe6712ba284585e7 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 11:46:32 -0700 Subject: [PATCH 20/32] add build bloblang string to transformer generator --- .../benthos/transformers/generator_utils.go | 54 +++++++++++----- .../neosync_transformer_generator.go | 28 +++++++++ .../gen-benthos-configs/processors.go | 63 ++++++++++++++----- 3 files changed, 113 insertions(+), 32 deletions(-) diff --git a/worker/pkg/benthos/transformers/generator_utils.go b/worker/pkg/benthos/transformers/generator_utils.go index 79aa96bd80..6c64d4bc0f 100644 --- a/worker/pkg/benthos/transformers/generator_utils.go +++ b/worker/pkg/benthos/transformers/generator_utils.go @@ -35,12 +35,13 @@ func toCamelCase(snake string) string { } type BenthosSpecParam struct { - Name string - TypeStr string - IsOptional bool - HasDefault bool - Default string - Description string + Name string + BloblangName string + TypeStr string + IsOptional bool + HasDefault bool + Default string + Description string } type BenthosSpec struct { @@ -53,8 +54,9 @@ type BenthosSpec struct { } type ParsedBenthosSpec struct { - Params []*BenthosSpecParam - SpecDescription string + Params []*BenthosSpecParam + BloblangFuncName string + SpecDescription string } func ExtractBenthosSpec(fileSet *token.FileSet) ([]*BenthosSpec, error) { @@ -139,24 +141,44 @@ func ParseBloblangSpec(benthosSpec *BenthosSpec) (*ParsedBenthosSpec, error) { } } param := &BenthosSpecParam{ - TypeStr: lowercaseFirst(matches[1]), - Name: toCamelCase(matches[2]), - IsOptional: strings.Contains(line, ".Optional()"), - HasDefault: defaultVal != "", - Default: defaultVal, - Description: description, + TypeStr: lowercaseFirst(matches[1]), + Name: toCamelCase(matches[2]), + BloblangName: matches[2], + IsOptional: strings.Contains(line, ".Optional()"), + HasDefault: defaultVal != "", + Default: defaultVal, + Description: description, } params = append(params, param) } } } + bloblangFuncName, err := extractBloblangFunctionName(benthosSpecStr) + if err != nil { + return nil, err + } + return &ParsedBenthosSpec{ - Params: params, - SpecDescription: specDescription, + BloblangFuncName: bloblangFuncName, + Params: params, + SpecDescription: specDescription, }, nil } +func extractBloblangFunctionName(input string) (string, error) { + // Looks for bloblang.RegisterFunctionV2 and captures the function name in quotes + re := regexp.MustCompile(`bloblang\.RegisterFunctionV2\("([^"]+)"`) + + matches := re.FindStringSubmatch(input) + + if len(matches) > 1 { + return matches[1], nil + } + + return "", fmt.Errorf("bloblang function name not found") +} + func lowercaseFirst(s string) string { if s == "" { return s diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 8aa00921a2..39bf83feed 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -65,6 +65,7 @@ func main() { } tf.Params = p.Params tf.Description = p.SpecDescription + tf.BloblangFuncName = p.BloblangFuncName } for _, tf := range transformerFuncs { @@ -166,6 +167,33 @@ func New{{.StructName}}Opts( }, nil } +func (o *{{.StructName}}Opts) BuildBloblangString(valuePath string) string { + fnStr := []string{ + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} + {{- if eq $param.Name "value" }}"value:this.%s",{{ else }} "{{$param.BloblangName}}:%v", {{ end }} + {{- end }} + } + params := []any{ + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} + {{- if eq $param.Name "value" }}valuePath,{{ else }} o.{{$param.Name}}, {{ end }} + {{- end }} + } + + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "value" }}{{ continue }}{{ end }} + {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} + {{- if $param.IsOptional }} + if o.{{$param.Name}} != nil { + fnStr = append(fnStr, "{{$param.BloblangName}}:%v") + } + {{- end }} + + template := fmt.Sprintf("{{ .FunctInfo.BloblangFuncName }}(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *{{.StructName}}) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "{{.FunctInfo.Name}}", diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go index 13b643759d..527b098f16 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go @@ -569,9 +569,6 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s pl := col.Transformer.Config.GetTransformFirstNameConfig().GetPreserveLength() return fmt.Sprintf("transform_first_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FLOAT64: - rMin := col.Transformer.Config.GetTransformFloat64Config().GetRandomizationRangeMin() - rMax := col.Transformer.Config.GetTransformFloat64Config().GetRandomizationRangeMax() - var precision *int64 if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { newPrecision := int64(*colInfo.NumericPrecision) @@ -583,20 +580,54 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s newScale := int64(*colInfo.NumericScale) scale = &newScale } - - fnStr := []string{"value:this.%s", "randomization_range_min:%f", "randomization_range_max:%f"} - params := []any{formattedColPath, rMin, rMax} - - if precision != nil { - fnStr = append(fnStr, "precision:%d") - params = append(params, *precision) - } - if scale != nil { - fnStr = append(fnStr, "scale:%d") - params = append(params, *scale) + opts, err := transformers.NewTransformFloat64OptsFromConfig(col.Transformer.Config.GetTransformFloat64Config(), scale, precision) + if err != nil { + return "", err } - template := fmt.Sprintf(`transform_float64(%s)`, strings.Join(fnStr, ", ")) - return fmt.Sprintf(template, params...), nil + return opts.BuildBenthosString(formattedColPath), nil + // config := col.Transformer.Config.GetTransformFloat64Config() + // var randomMin *float64 + // if config != nil && config.RandomizationRangeMin != nil { + // randomMin = config.RandomizationRangeMin + // } + // var randomMax *float64 + // if config != nil && config.RandomizationRangeMax != nil { + // randomMax = config.RandomizationRangeMax + // } + + // var precision *int64 + // if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { + // newPrecision := int64(*colInfo.NumericPrecision) + // precision = &newPrecision + // } + + // var scale *int64 + // if colInfo != nil && colInfo.NumericScale != nil && *colInfo.NumericScale >= 0 { + // newScale := int64(*colInfo.NumericScale) + // scale = &newScale + // } + + // fnStr := []string{"value:this.%s"} + // params := []any{formattedColPath} + + // if precision != nil { + // fnStr = append(fnStr, "precision:%d") + // params = append(params, *precision) + // } + // if scale != nil { + // fnStr = append(fnStr, "scale:%d") + // params = append(params, *scale) + // } + // if randomMin != nil { + // fnStr = append(fnStr, "randomization_range_min:%f") + // params = append(params, *randomMin) + // } + // if randomMax != nil { + // fnStr = append(fnStr, "randomization_range_max:%f") + // params = append(params, *randomMax) + // } + // template := fmt.Sprintf(`transform_float64(%s)`, strings.Join(fnStr, ", ")) + // return fmt.Sprintf(template, params...), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FULL_NAME: pl := col.Transformer.Config.GetTransformFullNameConfig().GetPreserveLength() return fmt.Sprintf("transform_full_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil From dfb3c69e7b3d1497a11b0793fc28dcae5fe502dc Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 11:47:25 -0700 Subject: [PATCH 21/32] forgot file --- worker/pkg/benthos/transformers/generator_utils.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/worker/pkg/benthos/transformers/generator_utils.go b/worker/pkg/benthos/transformers/generator_utils.go index 6c64d4bc0f..a7fa5224a7 100644 --- a/worker/pkg/benthos/transformers/generator_utils.go +++ b/worker/pkg/benthos/transformers/generator_utils.go @@ -45,12 +45,13 @@ type BenthosSpecParam struct { } type BenthosSpec struct { - Name string - Description string - Example string - Params []*BenthosSpecParam - Type string - SourceFile string + Name string + BloblangFuncName string + Description string + Example string + Params []*BenthosSpecParam + Type string + SourceFile string } type ParsedBenthosSpec struct { From d0c97cca810be37b515fa9c89d8d538af28c6565 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 11:59:28 -0700 Subject: [PATCH 22/32] update generator to get bloblang function name --- worker/pkg/benthos/transformers/generator_utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/worker/pkg/benthos/transformers/generator_utils.go b/worker/pkg/benthos/transformers/generator_utils.go index a7fa5224a7..a0441d6732 100644 --- a/worker/pkg/benthos/transformers/generator_utils.go +++ b/worker/pkg/benthos/transformers/generator_utils.go @@ -114,7 +114,8 @@ func ParseBloblangSpec(benthosSpec *BenthosSpec) (*ParsedBenthosSpec, error) { start = true benthosSpecStr += strings.TrimSpace(fileScanner.Text()) } else if start { - if strings.Contains(line, ":=") { + if strings.Contains(line, "bloblang.RegisterFunctionV2") { + benthosSpecStr += strings.TrimSpace(fileScanner.Text()) break } benthosSpecStr += strings.TrimSpace(fileScanner.Text()) From 0c651916e9ac485e0f020eade211f7c1f6bed0f1 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 12:04:16 -0700 Subject: [PATCH 23/32] ok --- .../neosync_transformer_generator.go | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 39bf83feed..ebe5d51675 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -171,27 +171,24 @@ func (o *{{.StructName}}Opts) BuildBloblangString(valuePath string) string { fnStr := []string{ {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} - {{- if eq $param.Name "value" }}"value:this.%s",{{ else }} "{{$param.BloblangName}}:%v", {{ end }} + {{- if eq $param.Name "value" }} + "value:this.%s", + {{- else }} + "{{$param.BloblangName}}:%v", {{- end }} - } - params := []any{ - {{- range $index, $param := .FunctInfo.Params }} - {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} - {{- if eq $param.Name "value" }}valuePath,{{ else }} o.{{$param.Name}}, {{ end }} {{- end }} } + params := []any{ {{- range $index, $param := .FunctInfo.Params }} - {{- if eq $param.Name "value" }}{{ continue }}{{ end }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} - {{- if $param.IsOptional }} - if o.{{$param.Name}} != nil { - fnStr = append(fnStr, "{{$param.BloblangName}}:%v") - } + {{- if eq $param.Name "value" }} + valuePath, + {{- else }} + o.{{$param.Name}}, {{- end }} - - template := fmt.Sprintf("{{ .FunctInfo.BloblangFuncName }}(%s)", strings.Join(fnStr, ", ")) - return fmt.Sprintf(template, params...) + {{- end }} + } } func (t *{{.StructName}}) GetJsTemplateData() (*TemplateData, error) { From e90007a2dc2ff6922f2be6a9c469b2076eb3c02b Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 12:12:13 -0700 Subject: [PATCH 24/32] generator working --- .../neosync_transformer_generator.go | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index ebe5d51675..4694ae623a 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -97,6 +97,7 @@ const codeTemplate = ` package {{.PackageName}} import ( + "strings" {{- if eq .ImportFmt true}} "fmt" {{ end }} @@ -167,7 +168,11 @@ func New{{.StructName}}Opts( }, nil } -func (o *{{.StructName}}Opts) BuildBloblangString(valuePath string) string { +func (o *{{.StructName}}Opts) BuildBloblangString( +{{- if .FunctInfo.IsTransformer }} + valuePath string, +{{- end }} +) string { fnStr := []string{ {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} @@ -189,6 +194,19 @@ func (o *{{.StructName}}Opts) BuildBloblangString(valuePath string) string { {{- end }} {{- end }} } + + {{- range $index, $param := .FunctInfo.Params }} + {{- if eq $param.Name "value" }}{{ continue }}{{ end }} + {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} + {{- if $param.IsOptional }} + if o.{{$param.Name}} != nil { + fnStr = append(fnStr, "{{$param.BloblangName}}:%v") + } + {{- end }} + {{- end }} + + template := fmt.Sprintf("{{ .FunctInfo.BloblangFuncName }}(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) } func (t *{{.StructName}}) GetJsTemplateData() (*TemplateData, error) { @@ -248,12 +266,13 @@ func (t *{{.StructName}}) ParseOptions(opts map[string]any) (any, error) { ` type TemplateData struct { - SourceFile string - PackageName string - FunctInfo transformers.BenthosSpec - StructName string - ImportFmt bool - HasSeedParam bool + SourceFile string + PackageName string + FunctInfo transformers.BenthosSpec + StructName string + ImportFmt bool + HasSeedParam bool + IsTransformer bool } func generateCode(pkgName string, funcInfo *transformers.BenthosSpec) (string, error) { @@ -271,6 +290,9 @@ func generateCode(pkgName string, funcInfo *transformers.BenthosSpec) (string, e if !p.IsOptional && !p.HasDefault { data.ImportFmt = true } + if p.Name == "value" { + data.IsTransformer = true + } } t := template.Must(template.New("neosyncTransformerImpl").Parse(codeTemplate)) var out bytes.Buffer From 05d7bcd4be22302881fa58dd155e80565a7d47e0 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 12:13:15 -0700 Subject: [PATCH 25/32] fix generator import --- .../pkg/benthos/transformers/neosync_transformer_generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 4694ae623a..301e0ec27b 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -169,7 +169,7 @@ func New{{.StructName}}Opts( } func (o *{{.StructName}}Opts) BuildBloblangString( -{{- if .FunctInfo.IsTransformer }} +{{- if .IsTransformer }} valuePath string, {{- end }} ) string { From 3c3b3ab529c3459733cdfbee400392f101a2637b Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 12:14:10 -0700 Subject: [PATCH 26/32] fix generator import --- .../benthos/transformers/neosync_transformer_generator.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 301e0ec27b..9e9d1de887 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -98,9 +98,7 @@ package {{.PackageName}} import ( "strings" - {{- if eq .ImportFmt true}} "fmt" - {{ end }} {{- if eq .HasSeedParam true}} transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -270,7 +268,6 @@ type TemplateData struct { PackageName string FunctInfo transformers.BenthosSpec StructName string - ImportFmt bool HasSeedParam bool IsTransformer bool } @@ -285,10 +282,6 @@ func generateCode(pkgName string, funcInfo *transformers.BenthosSpec) (string, e for _, p := range funcInfo.Params { if p.Name == "seed" { data.HasSeedParam = true - data.ImportFmt = true - } - if !p.IsOptional && !p.HasDefault { - data.ImportFmt = true } if p.Name == "value" { data.IsTransformer = true From 795c11ab9680447dd272bc1a38491a5b4fcc36ab Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 12:15:24 -0700 Subject: [PATCH 27/32] update gen files to have bloblang function --- .../benthos/transformers/gen_generate_bool.go | 14 ++++++++- .../transformers/gen_generate_card_number.go | 16 +++++++++- .../transformers/gen_generate_categorical.go | 16 +++++++++- .../benthos/transformers/gen_generate_city.go | 16 +++++++++- .../transformers/gen_generate_country.go | 16 +++++++++- .../transformers/gen_generate_email.go | 18 ++++++++++- .../transformers/gen_generate_first_name.go | 16 +++++++++- .../transformers/gen_generate_float.go | 30 +++++++++++++++++- .../transformers/gen_generate_full_address.go | 16 +++++++++- .../transformers/gen_generate_full_name.go | 16 +++++++++- .../transformers/gen_generate_gender.go | 18 ++++++++++- .../transformers/gen_generate_int64.go | 20 +++++++++++- .../gen_generate_int64_phone_number.go | 14 ++++++++- ...gen_generate_international_phone_number.go | 18 ++++++++++- .../transformers/gen_generate_last_name.go | 16 +++++++++- .../gen_generate_random_string.go | 18 ++++++++++- .../transformers/gen_generate_sha256hash.go | 14 +++++++++ .../benthos/transformers/gen_generate_ssn.go | 14 ++++++++- .../transformers/gen_generate_state.go | 16 +++++++++- .../gen_generate_street_address.go | 16 +++++++++- .../gen_generate_string_phone_number.go | 18 ++++++++++- .../gen_generate_unix_timestamp.go | 14 ++++++++- .../transformers/gen_generate_username.go | 16 +++++++++- .../gen_generate_utc_timestamp.go | 14 ++++++++- .../benthos/transformers/gen_generate_uuid.go | 16 ++++++++++ .../transformers/gen_generate_zipcode.go | 14 ++++++++- .../gen_transform_character_scramble.go | 22 ++++++++++++- .../gen_transform_e164_phone_number.go | 21 ++++++++++++- .../transformers/gen_transform_email.go | 29 ++++++++++++++++- .../transformers/gen_transform_first_name.go | 21 ++++++++++++- .../transformers/gen_transform_float.go | 31 ++++++++++++++++++- .../transformers/gen_transform_full_name.go | 21 ++++++++++++- .../transformers/gen_transform_int64.go | 21 ++++++++++++- .../gen_transform_int64_phone_number.go | 19 +++++++++++- .../transformers/gen_transform_lastname.go | 21 ++++++++++++- .../transformers/gen_transform_string.go | 23 +++++++++++++- .../gen_transform_string_phone_number.go | 21 ++++++++++++- 37 files changed, 645 insertions(+), 35 deletions(-) diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index 2bdc7d61c1..d88db24c0d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateBoolOpts( }, nil } +func (o *GenerateBoolOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_bool(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateBool) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateBool", diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index 98dc8ee0c9..15a362d0ae 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateCardNumberOpts( }, nil } +func (o *GenerateCardNumberOpts) BuildBloblangString( +) string { + fnStr := []string{ + "valid_luhn:%v", + } + + params := []any{ + o.validLuhn, + } + + template := fmt.Sprintf("generate_card_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateCardNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateCardNumber", diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index 582d0b0b88..e0602ef309 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateCategoricalOpts( }, nil } +func (o *GenerateCategoricalOpts) BuildBloblangString( +) string { + fnStr := []string{ + "categories:%v", + } + + params := []any{ + o.categories, + } + + template := fmt.Sprintf("generate_categorical(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateCategorical) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateCategorical", diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index e3adbed396..8df7034dc6 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateCityOpts( }, nil } +func (o *GenerateCityOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_city(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateCity) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateCity", diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index 4abd4caf9a..83929badda 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateCountryOpts( }, nil } +func (o *GenerateCountryOpts) BuildBloblangString( +) string { + fnStr := []string{ + "generate_full_name:%v", + } + + params := []any{ + o.generateFullName, + } + + template := fmt.Sprintf("generate_country(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateCountry) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateCountry", diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index cf6debb37d..4b8c5e245c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,22 @@ func NewGenerateEmailOpts( }, nil } +func (o *GenerateEmailOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + "email_type:%v", + } + + params := []any{ + o.maxLength, + o.emailType, + } + + template := fmt.Sprintf("generate_email(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateEmail) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateEmail", diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index 397f33289b..02102b2691 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateFirstNameOpts( }, nil } +func (o *GenerateFirstNameOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_first_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateFirstName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateFirstName", diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index d16fea18ad..6448703c40 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -66,6 +66,34 @@ func NewGenerateFloat64Opts( }, nil } +func (o *GenerateFloat64Opts) BuildBloblangString( +) string { + fnStr := []string{ + "randomize_sign:%v", + "min:%v", + "max:%v", + "precision:%v", + "scale:%v", + } + + params := []any{ + o.randomizeSign, + o.min, + o.max, + o.precision, + o.scale, + } + if o.precision != nil { + fnStr = append(fnStr, "precision:%v") + } + if o.scale != nil { + fnStr = append(fnStr, "scale:%v") + } + + template := fmt.Sprintf("generate_float64(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateFloat64) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateFloat64", diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index 57725e83fb..7e5a205502 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateFullAddressOpts( }, nil } +func (o *GenerateFullAddressOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_full_address(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateFullAddress) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateFullAddress", diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index 329173951c..588483ce2f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateFullNameOpts( }, nil } +func (o *GenerateFullNameOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_full_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateFullName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateFullName", diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index c03848454d..277eaf5d5f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,22 @@ func NewGenerateGenderOpts( }, nil } +func (o *GenerateGenderOpts) BuildBloblangString( +) string { + fnStr := []string{ + "abbreviate:%v", + "max_length:%v", + } + + params := []any{ + o.abbreviate, + o.maxLength, + } + + template := fmt.Sprintf("generate_gender(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateGender) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateGender", diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index 8c25f9ad37..0d8a0c7927 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -60,6 +60,24 @@ func NewGenerateInt64Opts( }, nil } +func (o *GenerateInt64Opts) BuildBloblangString( +) string { + fnStr := []string{ + "randomize_sign:%v", + "min:%v", + "max:%v", + } + + params := []any{ + o.randomizeSign, + o.min, + o.max, + } + + template := fmt.Sprintf("generate_int64(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateInt64) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateInt64", diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index 4b1cda478e..3466c4777d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateInt64PhoneNumberOpts( }, nil } +func (o *GenerateInt64PhoneNumberOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_int64_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateInt64PhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateInt64PhoneNumber", diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index b9934eddb4..ed93aa44e2 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,22 @@ func NewGenerateInternationalPhoneNumberOpts( }, nil } +func (o *GenerateInternationalPhoneNumberOpts) BuildBloblangString( +) string { + fnStr := []string{ + "min:%v", + "max:%v", + } + + params := []any{ + o.min, + o.max, + } + + template := fmt.Sprintf("generate_e164_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateInternationalPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateInternationalPhoneNumber", diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index 0bf3ddea59..5baf85bbb7 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateLastNameOpts( }, nil } +func (o *GenerateLastNameOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_last_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateLastName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateLastName", diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index 517881c61d..4a11649bbb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,22 @@ func NewGenerateRandomStringOpts( }, nil } +func (o *GenerateRandomStringOpts) BuildBloblangString( +) string { + fnStr := []string{ + "min:%v", + "max:%v", + } + + params := []any{ + o.min, + o.max, + } + + template := fmt.Sprintf("generate_string(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateRandomString) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateRandomString", diff --git a/worker/pkg/benthos/transformers/gen_generate_sha256hash.go b/worker/pkg/benthos/transformers/gen_generate_sha256hash.go index 70f225ff5e..5ef7890fc9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_sha256hash.go +++ b/worker/pkg/benthos/transformers/gen_generate_sha256hash.go @@ -5,6 +5,8 @@ package transformers import ( + "strings" + "fmt" ) type GenerateSHA256Hash struct{} @@ -22,6 +24,18 @@ func NewGenerateSHA256HashOpts( }, nil } +func (o *GenerateSHA256HashOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_sha256hash(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateSHA256Hash) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateSHA256Hash", diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index 444c4dd79a..aa183ab918 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateSSNOpts( }, nil } +func (o *GenerateSSNOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_ssn(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateSSN) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateSSN", diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index 59d29160a1..6184ab11e8 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateStateOpts( }, nil } +func (o *GenerateStateOpts) BuildBloblangString( +) string { + fnStr := []string{ + "generate_full_name:%v", + } + + params := []any{ + o.generateFullName, + } + + template := fmt.Sprintf("generate_state(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateState) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateState", diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index 264fab4d9b..8a98d5a777 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateStreetAddressOpts( }, nil } +func (o *GenerateStreetAddressOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_street_address(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateStreetAddress) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateStreetAddress", diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index a8fbcffca3..a977df3ad9 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,22 @@ func NewGenerateStringPhoneNumberOpts( }, nil } +func (o *GenerateStringPhoneNumberOpts) BuildBloblangString( +) string { + fnStr := []string{ + "min:%v", + "max:%v", + } + + params := []any{ + o.min, + o.max, + } + + template := fmt.Sprintf("generate_string_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateStringPhoneNumber", diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index f95717afe5..fafa5dbaa2 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateUnixTimestampOpts( }, nil } +func (o *GenerateUnixTimestampOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_unixtimestamp(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateUnixTimestamp) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateUnixTimestamp", diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index 6df1991fa0..05f7a81479 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,20 @@ func NewGenerateUsernameOpts( }, nil } +func (o *GenerateUsernameOpts) BuildBloblangString( +) string { + fnStr := []string{ + "max_length:%v", + } + + params := []any{ + o.maxLength, + } + + template := fmt.Sprintf("generate_username(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateUsername) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateUsername", diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index cbd99cdb08..a3c8267422 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateUTCTimestampOpts( }, nil } +func (o *GenerateUTCTimestampOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_utctimestamp(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateUTCTimestamp) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateUTCTimestamp", diff --git a/worker/pkg/benthos/transformers/gen_generate_uuid.go b/worker/pkg/benthos/transformers/gen_generate_uuid.go index 3c6801badb..16fd60fbc1 100644 --- a/worker/pkg/benthos/transformers/gen_generate_uuid.go +++ b/worker/pkg/benthos/transformers/gen_generate_uuid.go @@ -5,6 +5,8 @@ package transformers import ( + "strings" + "fmt" ) type GenerateUUID struct{} @@ -30,6 +32,20 @@ func NewGenerateUUIDOpts( }, nil } +func (o *GenerateUUIDOpts) BuildBloblangString( +) string { + fnStr := []string{ + "include_hyphens:%v", + } + + params := []any{ + o.includeHyphens, + } + + template := fmt.Sprintf("generate_uuid(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateUUID) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateUUID", diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index 2e1ab3bfb9..7355a9cb3c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -36,6 +36,18 @@ func NewGenerateZipcodeOpts( }, nil } +func (o *GenerateZipcodeOpts) BuildBloblangString( +) string { + fnStr := []string{ + } + + params := []any{ + } + + template := fmt.Sprintf("generate_zipcode(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *GenerateZipcode) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "generateZipcode", diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 5228484eb2..4988662407 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -39,6 +39,26 @@ func NewTransformCharacterScrambleOpts( }, nil } +func (o *TransformCharacterScrambleOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "user_provided_regex:%v", + } + + params := []any{ + valuePath, + o.userProvidedRegex, + } + if o.userProvidedRegex != nil { + fnStr = append(fnStr, "user_provided_regex:%v") + } + + template := fmt.Sprintf("transform_character_scramble(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformCharacterScramble) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformCharacterScramble", diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index e73f81a9bc..6b27136ad4 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformE164PhoneNumberOpts( }, nil } +func (o *TransformE164PhoneNumberOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "preserve_length:%v", + "max_length:%v", + } + + params := []any{ + valuePath, + o.preserveLength, + o.maxLength, + } + + template := fmt.Sprintf("transform_e164_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformE164PhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformE164PhoneNumber", diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index 1415f339bb..9fd740c33f 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -84,6 +84,33 @@ func NewTransformEmailOpts( }, nil } +func (o *TransformEmailOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "preserve_length:%v", + "preserve_domain:%v", + "excluded_domains:%v", + "max_length:%v", + "email_type:%v", + "invalid_email_action:%v", + } + + params := []any{ + valuePath, + o.preserveLength, + o.preserveDomain, + o.excludedDomains, + o.maxLength, + o.emailType, + o.invalidEmailAction, + } + + template := fmt.Sprintf("transform_email(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformEmail) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformEmail", diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index ec28e22383..9a03b0a817 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformFirstNameOpts( }, nil } +func (o *TransformFirstNameOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "max_length:%v", + "value:this.%s", + "preserve_length:%v", + } + + params := []any{ + o.maxLength, + valuePath, + o.preserveLength, + } + + template := fmt.Sprintf("transform_first_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformFirstName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformFirstName", diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index 39642acfd7..549c935e1c 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -58,6 +58,35 @@ func NewTransformFloat64Opts( }, nil } +func (o *TransformFloat64Opts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "randomization_range_min:%v", + "randomization_range_max:%v", + "precision:%v", + "scale:%v", + } + + params := []any{ + valuePath, + o.randomizationRangeMin, + o.randomizationRangeMax, + o.precision, + o.scale, + } + if o.precision != nil { + fnStr = append(fnStr, "precision:%v") + } + if o.scale != nil { + fnStr = append(fnStr, "scale:%v") + } + + template := fmt.Sprintf("transform_float64(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformFloat64) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformFloat64", diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index c6c2792733..bf4ac07ce7 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformFullNameOpts( }, nil } +func (o *TransformFullNameOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "max_length:%v", + "value:this.%s", + "preserve_length:%v", + } + + params := []any{ + o.maxLength, + valuePath, + o.preserveLength, + } + + template := fmt.Sprintf("transform_full_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformFullName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformFullName", diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index c98491a801..c10b7755d9 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformInt64Opts( }, nil } +func (o *TransformInt64Opts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "randomization_range_min:%v", + "randomization_range_max:%v", + } + + params := []any{ + valuePath, + o.randomizationRangeMin, + o.randomizationRangeMax, + } + + template := fmt.Sprintf("transform_int64(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformInt64) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformInt64", diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index a51ed2a753..e73c00f013 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -44,6 +44,23 @@ func NewTransformInt64PhoneNumberOpts( }, nil } +func (o *TransformInt64PhoneNumberOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "preserve_length:%v", + } + + params := []any{ + valuePath, + o.preserveLength, + } + + template := fmt.Sprintf("transform_int64_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformInt64PhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformInt64PhoneNumber", diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index f2e5020291..c44d688036 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformLastNameOpts( }, nil } +func (o *TransformLastNameOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "max_length:%v", + "value:this.%s", + "preserve_length:%v", + } + + params := []any{ + o.maxLength, + valuePath, + o.preserveLength, + } + + template := fmt.Sprintf("transform_last_name(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformLastName) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformLastName", diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index a773e07485..97a08c2e1b 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -60,6 +60,27 @@ func NewTransformStringOpts( }, nil } +func (o *TransformStringOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "preserve_length:%v", + "min_length:%v", + "max_length:%v", + } + + params := []any{ + valuePath, + o.preserveLength, + o.minLength, + o.maxLength, + } + + template := fmt.Sprintf("transform_string(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformString) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformString", diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 65a0f6011e..3124bd3856 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -5,8 +5,8 @@ package transformers import ( + "strings" "fmt" - transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" "github.com/nucleuscloud/neosync/worker/pkg/rng" @@ -52,6 +52,25 @@ func NewTransformStringPhoneNumberOpts( }, nil } +func (o *TransformStringPhoneNumberOpts) BuildBloblangString( + valuePath string, +) string { + fnStr := []string{ + "value:this.%s", + "preserve_length:%v", + "max_length:%v", + } + + params := []any{ + valuePath, + o.preserveLength, + o.maxLength, + } + + template := fmt.Sprintf("transform_phone_number(%s)", strings.Join(fnStr, ", ")) + return fmt.Sprintf(template, params...) +} + func (t *TransformStringPhoneNumber) GetJsTemplateData() (*TemplateData, error) { return &TemplateData{ Name: "transformStringPhoneNumber", From 49533bb57b2e4737e6d90b2bd788e449a51f9b32 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 15:53:00 -0700 Subject: [PATCH 28/32] update transformer processors --- .../benthos/transformers/gen_generate_bool.go | 4 +- .../transformers/gen_generate_card_number.go | 10 +- .../transformers/gen_generate_categorical.go | 10 +- .../benthos/transformers/gen_generate_city.go | 10 +- .../transformers/gen_generate_country.go | 10 +- .../transformers/gen_generate_email.go | 16 +- .../transformers/gen_generate_first_name.go | 10 +- .../transformers/gen_generate_float.go | 28 +- .../transformers/gen_generate_full_address.go | 10 +- .../transformers/gen_generate_full_name.go | 10 +- .../transformers/gen_generate_gender.go | 16 +- .../transformers/gen_generate_int64.go | 22 +- .../gen_generate_int64_phone_number.go | 4 +- ...gen_generate_international_phone_number.go | 16 +- .../transformers/gen_generate_last_name.go | 10 +- .../gen_generate_random_string.go | 16 +- .../transformers/gen_generate_sha256hash.go | 4 +- .../benthos/transformers/gen_generate_ssn.go | 4 +- .../transformers/gen_generate_state.go | 10 +- .../gen_generate_street_address.go | 10 +- .../gen_generate_string_phone_number.go | 16 +- .../gen_generate_unix_timestamp.go | 4 +- .../transformers/gen_generate_username.go | 10 +- .../gen_generate_utc_timestamp.go | 4 +- .../benthos/transformers/gen_generate_uuid.go | 10 +- .../transformers/gen_generate_zipcode.go | 4 +- .../gen_transform_character_scramble.go | 13 +- .../gen_transform_e164_phone_number.go | 20 +- .../transformers/gen_transform_email.go | 44 ++- .../transformers/gen_transform_first_name.go | 20 +- .../transformers/gen_transform_float.go | 26 +- .../transformers/gen_transform_full_name.go | 20 +- .../transformers/gen_transform_int64.go | 20 +- .../gen_transform_int64_phone_number.go | 14 +- .../transformers/gen_transform_lastname.go | 20 +- .../transformers/gen_transform_string.go | 26 +- .../gen_transform_string_phone_number.go | 20 +- .../neosync_transformer_generator.go | 25 +- .../benthos/transformers/transform_email.go | 28 +- .../transformers/transform_email_test.go | 16 + .../benthos-builder_test.go | 207 +++++++++- .../gen-benthos-configs/processors.go | 360 +++++++++--------- .../activities/gen-benthos-configs/utils.go | 16 - 43 files changed, 725 insertions(+), 448 deletions(-) diff --git a/worker/pkg/benthos/transformers/gen_generate_bool.go b/worker/pkg/benthos/transformers/gen_generate_bool.go index d88db24c0d..faed0d4efb 100644 --- a/worker/pkg/benthos/transformers/gen_generate_bool.go +++ b/worker/pkg/benthos/transformers/gen_generate_bool.go @@ -44,7 +44,9 @@ func (o *GenerateBoolOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_bool(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_bool(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_card_number.go b/worker/pkg/benthos/transformers/gen_generate_card_number.go index 15a362d0ae..e2c22bc74d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_card_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_card_number.go @@ -28,7 +28,7 @@ func NewGenerateCardNumberOpts( validLuhnArg *bool, seedArg *int64, ) (*GenerateCardNumberOpts, error) { - validLuhn := bool(false) + validLuhn := bool(false) if validLuhnArg != nil { validLuhn = *validLuhnArg } @@ -47,14 +47,16 @@ func NewGenerateCardNumberOpts( func (o *GenerateCardNumberOpts) BuildBloblangString( ) string { fnStr := []string{ - "valid_luhn:%v", + "valid_luhn:%v", } params := []any{ - o.validLuhn, + o.validLuhn, } - template := fmt.Sprintf("generate_card_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_card_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_categorical.go b/worker/pkg/benthos/transformers/gen_generate_categorical.go index e0602ef309..85bfbf33d8 100644 --- a/worker/pkg/benthos/transformers/gen_generate_categorical.go +++ b/worker/pkg/benthos/transformers/gen_generate_categorical.go @@ -28,7 +28,7 @@ func NewGenerateCategoricalOpts( categoriesArg *string, seedArg *int64, ) (*GenerateCategoricalOpts, error) { - categories := string("ultimo,proximo,semper") + categories := string("ultimo,proximo,semper") if categoriesArg != nil { categories = *categoriesArg } @@ -47,14 +47,16 @@ func NewGenerateCategoricalOpts( func (o *GenerateCategoricalOpts) BuildBloblangString( ) string { fnStr := []string{ - "categories:%v", + "categories:%q", } params := []any{ - o.categories, + o.categories, } - template := fmt.Sprintf("generate_categorical(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_categorical(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_city.go b/worker/pkg/benthos/transformers/gen_generate_city.go index 8df7034dc6..ce84948925 100644 --- a/worker/pkg/benthos/transformers/gen_generate_city.go +++ b/worker/pkg/benthos/transformers/gen_generate_city.go @@ -28,7 +28,7 @@ func NewGenerateCityOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateCityOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateCityOpts( func (o *GenerateCityOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_city(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_city(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_country.go b/worker/pkg/benthos/transformers/gen_generate_country.go index 83929badda..179e2390e0 100644 --- a/worker/pkg/benthos/transformers/gen_generate_country.go +++ b/worker/pkg/benthos/transformers/gen_generate_country.go @@ -28,7 +28,7 @@ func NewGenerateCountryOpts( generateFullNameArg *bool, seedArg *int64, ) (*GenerateCountryOpts, error) { - generateFullName := bool(false) + generateFullName := bool(false) if generateFullNameArg != nil { generateFullName = *generateFullNameArg } @@ -47,14 +47,16 @@ func NewGenerateCountryOpts( func (o *GenerateCountryOpts) BuildBloblangString( ) string { fnStr := []string{ - "generate_full_name:%v", + "generate_full_name:%v", } params := []any{ - o.generateFullName, + o.generateFullName, } - template := fmt.Sprintf("generate_country(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_country(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_email.go b/worker/pkg/benthos/transformers/gen_generate_email.go index 4b8c5e245c..5d31d1011a 100644 --- a/worker/pkg/benthos/transformers/gen_generate_email.go +++ b/worker/pkg/benthos/transformers/gen_generate_email.go @@ -30,12 +30,12 @@ func NewGenerateEmailOpts( emailTypeArg *string, seedArg *int64, ) (*GenerateEmailOpts, error) { - maxLength := int64(100000) + maxLength := int64(100000) if maxLengthArg != nil { maxLength = *maxLengthArg } - emailType := string(GenerateEmailType_UuidV4.String()) + emailType := string(GenerateEmailType_UuidV4.String()) if emailTypeArg != nil { emailType = *emailTypeArg } @@ -55,16 +55,18 @@ func NewGenerateEmailOpts( func (o *GenerateEmailOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", - "email_type:%v", + "max_length:%v", + "email_type:%q", } params := []any{ - o.maxLength, - o.emailType, + o.maxLength, + o.emailType, } - template := fmt.Sprintf("generate_email(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_email(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_first_name.go b/worker/pkg/benthos/transformers/gen_generate_first_name.go index 02102b2691..5080c9e426 100644 --- a/worker/pkg/benthos/transformers/gen_generate_first_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_first_name.go @@ -28,7 +28,7 @@ func NewGenerateFirstNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateFirstNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateFirstNameOpts( func (o *GenerateFirstNameOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_first_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_first_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_float.go b/worker/pkg/benthos/transformers/gen_generate_float.go index 6448703c40..a6c1edb3ea 100644 --- a/worker/pkg/benthos/transformers/gen_generate_float.go +++ b/worker/pkg/benthos/transformers/gen_generate_float.go @@ -36,17 +36,17 @@ func NewGenerateFloat64Opts( scale *int64, seedArg *int64, ) (*GenerateFloat64Opts, error) { - randomizeSign := bool(false) + randomizeSign := bool(false) if randomizeSignArg != nil { randomizeSign = *randomizeSignArg } - min := float64(1) + min := float64(1) if minArg != nil { min = *minArg } - max := float64(10000) + max := float64(10000) if maxArg != nil { max = *maxArg } @@ -69,28 +69,28 @@ func NewGenerateFloat64Opts( func (o *GenerateFloat64Opts) BuildBloblangString( ) string { fnStr := []string{ - "randomize_sign:%v", - "min:%v", - "max:%v", - "precision:%v", - "scale:%v", + "randomize_sign:%v", + "min:%v", + "max:%v", } params := []any{ - o.randomizeSign, - o.min, - o.max, - o.precision, - o.scale, + o.randomizeSign, + o.min, + o.max, } + + if o.precision != nil { fnStr = append(fnStr, "precision:%v") + params = append(params, *o.precision) } if o.scale != nil { fnStr = append(fnStr, "scale:%v") + params = append(params, *o.scale) } - template := fmt.Sprintf("generate_float64(%s)", strings.Join(fnStr, ", ")) + template := fmt.Sprintf("generate_float64(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_full_address.go b/worker/pkg/benthos/transformers/gen_generate_full_address.go index 7e5a205502..84f318bb0c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_address.go @@ -28,7 +28,7 @@ func NewGenerateFullAddressOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateFullAddressOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateFullAddressOpts( func (o *GenerateFullAddressOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_full_address(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_full_address(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_full_name.go b/worker/pkg/benthos/transformers/gen_generate_full_name.go index 588483ce2f..8d8ae5bc8e 100644 --- a/worker/pkg/benthos/transformers/gen_generate_full_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_full_name.go @@ -28,7 +28,7 @@ func NewGenerateFullNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateFullNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateFullNameOpts( func (o *GenerateFullNameOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_full_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_full_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_gender.go b/worker/pkg/benthos/transformers/gen_generate_gender.go index 277eaf5d5f..b9efd73e5d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_gender.go +++ b/worker/pkg/benthos/transformers/gen_generate_gender.go @@ -30,12 +30,12 @@ func NewGenerateGenderOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateGenderOpts, error) { - abbreviate := bool(false) + abbreviate := bool(false) if abbreviateArg != nil { abbreviate = *abbreviateArg } - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -55,16 +55,18 @@ func NewGenerateGenderOpts( func (o *GenerateGenderOpts) BuildBloblangString( ) string { fnStr := []string{ - "abbreviate:%v", - "max_length:%v", + "abbreviate:%v", + "max_length:%v", } params := []any{ - o.abbreviate, - o.maxLength, + o.abbreviate, + o.maxLength, } - template := fmt.Sprintf("generate_gender(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_gender(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_int64.go b/worker/pkg/benthos/transformers/gen_generate_int64.go index 0d8a0c7927..c183b37db8 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64.go @@ -32,17 +32,17 @@ func NewGenerateInt64Opts( maxArg *int64, seedArg *int64, ) (*GenerateInt64Opts, error) { - randomizeSign := bool(false) + randomizeSign := bool(false) if randomizeSignArg != nil { randomizeSign = *randomizeSignArg } - min := int64(1) + min := int64(1) if minArg != nil { min = *minArg } - max := int64(10000) + max := int64(10000) if maxArg != nil { max = *maxArg } @@ -63,18 +63,20 @@ func NewGenerateInt64Opts( func (o *GenerateInt64Opts) BuildBloblangString( ) string { fnStr := []string{ - "randomize_sign:%v", - "min:%v", - "max:%v", + "randomize_sign:%v", + "min:%v", + "max:%v", } params := []any{ - o.randomizeSign, - o.min, - o.max, + o.randomizeSign, + o.min, + o.max, } - template := fmt.Sprintf("generate_int64(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_int64(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go index 3466c4777d..6a3fc1ebd4 100644 --- a/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_int64_phone_number.go @@ -44,7 +44,9 @@ func (o *GenerateInt64PhoneNumberOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_int64_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_int64_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go index ed93aa44e2..731c18bf2d 100644 --- a/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_international_phone_number.go @@ -30,12 +30,12 @@ func NewGenerateInternationalPhoneNumberOpts( maxArg *int64, seedArg *int64, ) (*GenerateInternationalPhoneNumberOpts, error) { - min := int64(9) + min := int64(9) if minArg != nil { min = *minArg } - max := int64(15) + max := int64(15) if maxArg != nil { max = *maxArg } @@ -55,16 +55,18 @@ func NewGenerateInternationalPhoneNumberOpts( func (o *GenerateInternationalPhoneNumberOpts) BuildBloblangString( ) string { fnStr := []string{ - "min:%v", - "max:%v", + "min:%v", + "max:%v", } params := []any{ - o.min, - o.max, + o.min, + o.max, } - template := fmt.Sprintf("generate_e164_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_e164_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_last_name.go b/worker/pkg/benthos/transformers/gen_generate_last_name.go index 5baf85bbb7..e12a6bd536 100644 --- a/worker/pkg/benthos/transformers/gen_generate_last_name.go +++ b/worker/pkg/benthos/transformers/gen_generate_last_name.go @@ -28,7 +28,7 @@ func NewGenerateLastNameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateLastNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateLastNameOpts( func (o *GenerateLastNameOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_last_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_last_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_random_string.go b/worker/pkg/benthos/transformers/gen_generate_random_string.go index 4a11649bbb..aa008bd6c8 100644 --- a/worker/pkg/benthos/transformers/gen_generate_random_string.go +++ b/worker/pkg/benthos/transformers/gen_generate_random_string.go @@ -30,12 +30,12 @@ func NewGenerateRandomStringOpts( maxArg *int64, seedArg *int64, ) (*GenerateRandomStringOpts, error) { - min := int64(1) + min := int64(1) if minArg != nil { min = *minArg } - max := int64(100) + max := int64(100) if maxArg != nil { max = *maxArg } @@ -55,16 +55,18 @@ func NewGenerateRandomStringOpts( func (o *GenerateRandomStringOpts) BuildBloblangString( ) string { fnStr := []string{ - "min:%v", - "max:%v", + "min:%v", + "max:%v", } params := []any{ - o.min, - o.max, + o.min, + o.max, } - template := fmt.Sprintf("generate_string(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_string(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_sha256hash.go b/worker/pkg/benthos/transformers/gen_generate_sha256hash.go index 5ef7890fc9..063801e6db 100644 --- a/worker/pkg/benthos/transformers/gen_generate_sha256hash.go +++ b/worker/pkg/benthos/transformers/gen_generate_sha256hash.go @@ -32,7 +32,9 @@ func (o *GenerateSHA256HashOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_sha256hash(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_sha256hash(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_ssn.go b/worker/pkg/benthos/transformers/gen_generate_ssn.go index aa183ab918..b079a90e8f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_ssn.go +++ b/worker/pkg/benthos/transformers/gen_generate_ssn.go @@ -44,7 +44,9 @@ func (o *GenerateSSNOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_ssn(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_ssn(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_state.go b/worker/pkg/benthos/transformers/gen_generate_state.go index 6184ab11e8..10faa54a4b 100644 --- a/worker/pkg/benthos/transformers/gen_generate_state.go +++ b/worker/pkg/benthos/transformers/gen_generate_state.go @@ -28,7 +28,7 @@ func NewGenerateStateOpts( generateFullNameArg *bool, seedArg *int64, ) (*GenerateStateOpts, error) { - generateFullName := bool(false) + generateFullName := bool(false) if generateFullNameArg != nil { generateFullName = *generateFullNameArg } @@ -47,14 +47,16 @@ func NewGenerateStateOpts( func (o *GenerateStateOpts) BuildBloblangString( ) string { fnStr := []string{ - "generate_full_name:%v", + "generate_full_name:%v", } params := []any{ - o.generateFullName, + o.generateFullName, } - template := fmt.Sprintf("generate_state(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_state(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_street_address.go b/worker/pkg/benthos/transformers/gen_generate_street_address.go index 8a98d5a777..7d175dd316 100644 --- a/worker/pkg/benthos/transformers/gen_generate_street_address.go +++ b/worker/pkg/benthos/transformers/gen_generate_street_address.go @@ -28,7 +28,7 @@ func NewGenerateStreetAddressOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateStreetAddressOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateStreetAddressOpts( func (o *GenerateStreetAddressOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_street_address(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_street_address(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go index a977df3ad9..3432df9c3c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_generate_string_phone_number.go @@ -30,12 +30,12 @@ func NewGenerateStringPhoneNumberOpts( maxArg *int64, seedArg *int64, ) (*GenerateStringPhoneNumberOpts, error) { - min := int64(9) + min := int64(9) if minArg != nil { min = *minArg } - max := int64(15) + max := int64(15) if maxArg != nil { max = *maxArg } @@ -55,16 +55,18 @@ func NewGenerateStringPhoneNumberOpts( func (o *GenerateStringPhoneNumberOpts) BuildBloblangString( ) string { fnStr := []string{ - "min:%v", - "max:%v", + "min:%v", + "max:%v", } params := []any{ - o.min, - o.max, + o.min, + o.max, } - template := fmt.Sprintf("generate_string_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_string_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go index fafa5dbaa2..1f3334c07c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_unix_timestamp.go @@ -44,7 +44,9 @@ func (o *GenerateUnixTimestampOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_unixtimestamp(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_unixtimestamp(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_username.go b/worker/pkg/benthos/transformers/gen_generate_username.go index 05f7a81479..1cb3e0cd3c 100644 --- a/worker/pkg/benthos/transformers/gen_generate_username.go +++ b/worker/pkg/benthos/transformers/gen_generate_username.go @@ -28,7 +28,7 @@ func NewGenerateUsernameOpts( maxLengthArg *int64, seedArg *int64, ) (*GenerateUsernameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -47,14 +47,16 @@ func NewGenerateUsernameOpts( func (o *GenerateUsernameOpts) BuildBloblangString( ) string { fnStr := []string{ - "max_length:%v", + "max_length:%v", } params := []any{ - o.maxLength, + o.maxLength, } - template := fmt.Sprintf("generate_username(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_username(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go index a3c8267422..8df5bf4c0f 100644 --- a/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go +++ b/worker/pkg/benthos/transformers/gen_generate_utc_timestamp.go @@ -44,7 +44,9 @@ func (o *GenerateUTCTimestampOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_utctimestamp(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_utctimestamp(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_uuid.go b/worker/pkg/benthos/transformers/gen_generate_uuid.go index 16fd60fbc1..0a0cbb1dd3 100644 --- a/worker/pkg/benthos/transformers/gen_generate_uuid.go +++ b/worker/pkg/benthos/transformers/gen_generate_uuid.go @@ -22,7 +22,7 @@ func NewGenerateUUID() *GenerateUUID { func NewGenerateUUIDOpts( includeHyphensArg *bool, ) (*GenerateUUIDOpts, error) { - includeHyphens := bool(true) + includeHyphens := bool(true) if includeHyphensArg != nil { includeHyphens = *includeHyphensArg } @@ -35,14 +35,16 @@ func NewGenerateUUIDOpts( func (o *GenerateUUIDOpts) BuildBloblangString( ) string { fnStr := []string{ - "include_hyphens:%v", + "include_hyphens:%v", } params := []any{ - o.includeHyphens, + o.includeHyphens, } - template := fmt.Sprintf("generate_uuid(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_uuid(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_generate_zipcode.go b/worker/pkg/benthos/transformers/gen_generate_zipcode.go index 7355a9cb3c..7fdc6d45e2 100644 --- a/worker/pkg/benthos/transformers/gen_generate_zipcode.go +++ b/worker/pkg/benthos/transformers/gen_generate_zipcode.go @@ -44,7 +44,9 @@ func (o *GenerateZipcodeOpts) BuildBloblangString( params := []any{ } - template := fmt.Sprintf("generate_zipcode(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("generate_zipcode(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go index 4988662407..31a7c3d1d5 100644 --- a/worker/pkg/benthos/transformers/gen_transform_character_scramble.go +++ b/worker/pkg/benthos/transformers/gen_transform_character_scramble.go @@ -43,19 +43,20 @@ func (o *TransformCharacterScrambleOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "user_provided_regex:%v", + "value:this.%s", } params := []any{ - valuePath, - o.userProvidedRegex, + valuePath, } + + if o.userProvidedRegex != nil { - fnStr = append(fnStr, "user_provided_regex:%v") + fnStr = append(fnStr, "user_provided_regex:%q") + params = append(params, *o.userProvidedRegex) } - template := fmt.Sprintf("transform_character_scramble(%s)", strings.Join(fnStr, ", ")) + template := fmt.Sprintf("transform_character_scramble(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go index 6b27136ad4..56870b7065 100644 --- a/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_e164_phone_number.go @@ -30,12 +30,12 @@ func NewTransformE164PhoneNumberOpts( maxLengthArg *int64, seedArg *int64, ) (*TransformE164PhoneNumberOpts, error) { - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } - maxLength := int64(15) + maxLength := int64(15) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -56,18 +56,20 @@ func (o *TransformE164PhoneNumberOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "preserve_length:%v", - "max_length:%v", + "value:this.%s", + "preserve_length:%v", + "max_length:%v", } params := []any{ - valuePath, - o.preserveLength, - o.maxLength, + valuePath, + o.preserveLength, + o.maxLength, } - template := fmt.Sprintf("transform_e164_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_e164_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_email.go b/worker/pkg/benthos/transformers/gen_transform_email.go index 9fd740c33f..b98577d440 100644 --- a/worker/pkg/benthos/transformers/gen_transform_email.go +++ b/worker/pkg/benthos/transformers/gen_transform_email.go @@ -38,22 +38,22 @@ func NewTransformEmailOpts( emailTypeArg *string, invalidEmailActionArg *string, ) (*TransformEmailOpts, error) { - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } - preserveDomain := bool(false) + preserveDomain := bool(false) if preserveDomainArg != nil { preserveDomain = *preserveDomainArg } - excludedDomains := any([]any{}) + var excludedDomains any if excludedDomainsArg != nil { excludedDomains = *excludedDomainsArg } - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -63,12 +63,12 @@ func NewTransformEmailOpts( return nil, fmt.Errorf("unable to generate seed: %w", err) } - emailType := string(GenerateEmailType_UuidV4.String()) + emailType := string(GenerateEmailType_UuidV4.String()) if emailTypeArg != nil { emailType = *emailTypeArg } - invalidEmailAction := string(InvalidEmailAction_Reject.String()) + invalidEmailAction := string(InvalidEmailAction_Reject.String()) if invalidEmailActionArg != nil { invalidEmailAction = *invalidEmailActionArg } @@ -88,26 +88,28 @@ func (o *TransformEmailOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "preserve_length:%v", - "preserve_domain:%v", - "excluded_domains:%v", - "max_length:%v", - "email_type:%v", - "invalid_email_action:%v", + "value:this.%s", + "preserve_length:%v", + "preserve_domain:%v", + "excluded_domains:%v", + "max_length:%v", + "email_type:%q", + "invalid_email_action:%q", } params := []any{ - valuePath, - o.preserveLength, - o.preserveDomain, - o.excludedDomains, - o.maxLength, - o.emailType, - o.invalidEmailAction, + valuePath, + o.preserveLength, + o.preserveDomain, + o.excludedDomains, + o.maxLength, + o.emailType, + o.invalidEmailAction, } - template := fmt.Sprintf("transform_email(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_email(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_first_name.go b/worker/pkg/benthos/transformers/gen_transform_first_name.go index 9a03b0a817..f5f673d2d8 100644 --- a/worker/pkg/benthos/transformers/gen_transform_first_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_first_name.go @@ -30,12 +30,12 @@ func NewTransformFirstNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformFirstNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } @@ -56,18 +56,20 @@ func (o *TransformFirstNameOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "max_length:%v", - "value:this.%s", - "preserve_length:%v", + "max_length:%v", + "value:this.%s", + "preserve_length:%v", } params := []any{ - o.maxLength, - valuePath, - o.preserveLength, + o.maxLength, + valuePath, + o.preserveLength, } - template := fmt.Sprintf("transform_first_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_first_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_float.go b/worker/pkg/benthos/transformers/gen_transform_float.go index 549c935e1c..938cf2a2ac 100644 --- a/worker/pkg/benthos/transformers/gen_transform_float.go +++ b/worker/pkg/benthos/transformers/gen_transform_float.go @@ -34,12 +34,12 @@ func NewTransformFloat64Opts( scale *int64, seedArg *int64, ) (*TransformFloat64Opts, error) { - randomizationRangeMin := float64(1) + randomizationRangeMin := float64(1) if randomizationRangeMinArg != nil { randomizationRangeMin = *randomizationRangeMinArg } - randomizationRangeMax := float64(10000) + randomizationRangeMax := float64(10000) if randomizationRangeMaxArg != nil { randomizationRangeMax = *randomizationRangeMaxArg } @@ -62,28 +62,28 @@ func (o *TransformFloat64Opts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "randomization_range_min:%v", - "randomization_range_max:%v", - "precision:%v", - "scale:%v", + "value:this.%s", + "randomization_range_min:%v", + "randomization_range_max:%v", } params := []any{ - valuePath, - o.randomizationRangeMin, - o.randomizationRangeMax, - o.precision, - o.scale, + valuePath, + o.randomizationRangeMin, + o.randomizationRangeMax, } + + if o.precision != nil { fnStr = append(fnStr, "precision:%v") + params = append(params, *o.precision) } if o.scale != nil { fnStr = append(fnStr, "scale:%v") + params = append(params, *o.scale) } - template := fmt.Sprintf("transform_float64(%s)", strings.Join(fnStr, ", ")) + template := fmt.Sprintf("transform_float64(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_full_name.go b/worker/pkg/benthos/transformers/gen_transform_full_name.go index bf4ac07ce7..49217a7314 100644 --- a/worker/pkg/benthos/transformers/gen_transform_full_name.go +++ b/worker/pkg/benthos/transformers/gen_transform_full_name.go @@ -30,12 +30,12 @@ func NewTransformFullNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformFullNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } @@ -56,18 +56,20 @@ func (o *TransformFullNameOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "max_length:%v", - "value:this.%s", - "preserve_length:%v", + "max_length:%v", + "value:this.%s", + "preserve_length:%v", } params := []any{ - o.maxLength, - valuePath, - o.preserveLength, + o.maxLength, + valuePath, + o.preserveLength, } - template := fmt.Sprintf("transform_full_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_full_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_int64.go b/worker/pkg/benthos/transformers/gen_transform_int64.go index c10b7755d9..a3424225ed 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64.go @@ -30,12 +30,12 @@ func NewTransformInt64Opts( randomizationRangeMaxArg *int64, seedArg *int64, ) (*TransformInt64Opts, error) { - randomizationRangeMin := int64(1) + randomizationRangeMin := int64(1) if randomizationRangeMinArg != nil { randomizationRangeMin = *randomizationRangeMinArg } - randomizationRangeMax := int64(10000) + randomizationRangeMax := int64(10000) if randomizationRangeMaxArg != nil { randomizationRangeMax = *randomizationRangeMaxArg } @@ -56,18 +56,20 @@ func (o *TransformInt64Opts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "randomization_range_min:%v", - "randomization_range_max:%v", + "value:this.%s", + "randomization_range_min:%v", + "randomization_range_max:%v", } params := []any{ - valuePath, - o.randomizationRangeMin, - o.randomizationRangeMax, + valuePath, + o.randomizationRangeMin, + o.randomizationRangeMax, } - template := fmt.Sprintf("transform_int64(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_int64(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go index e73c00f013..c3d44c711e 100644 --- a/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_int64_phone_number.go @@ -28,7 +28,7 @@ func NewTransformInt64PhoneNumberOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformInt64PhoneNumberOpts, error) { - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } @@ -48,16 +48,18 @@ func (o *TransformInt64PhoneNumberOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "preserve_length:%v", + "value:this.%s", + "preserve_length:%v", } params := []any{ - valuePath, - o.preserveLength, + valuePath, + o.preserveLength, } - template := fmt.Sprintf("transform_int64_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_int64_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_lastname.go b/worker/pkg/benthos/transformers/gen_transform_lastname.go index c44d688036..e6e72f3a26 100644 --- a/worker/pkg/benthos/transformers/gen_transform_lastname.go +++ b/worker/pkg/benthos/transformers/gen_transform_lastname.go @@ -30,12 +30,12 @@ func NewTransformLastNameOpts( preserveLengthArg *bool, seedArg *int64, ) (*TransformLastNameOpts, error) { - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } @@ -56,18 +56,20 @@ func (o *TransformLastNameOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "max_length:%v", - "value:this.%s", - "preserve_length:%v", + "max_length:%v", + "value:this.%s", + "preserve_length:%v", } params := []any{ - o.maxLength, - valuePath, - o.preserveLength, + o.maxLength, + valuePath, + o.preserveLength, } - template := fmt.Sprintf("transform_last_name(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_last_name(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_string.go b/worker/pkg/benthos/transformers/gen_transform_string.go index 97a08c2e1b..b36aaf3dd1 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string.go +++ b/worker/pkg/benthos/transformers/gen_transform_string.go @@ -32,17 +32,17 @@ func NewTransformStringOpts( maxLengthArg *int64, seedArg *int64, ) (*TransformStringOpts, error) { - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } - minLength := int64(1) + minLength := int64(1) if minLengthArg != nil { minLength = *minLengthArg } - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -64,20 +64,22 @@ func (o *TransformStringOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "preserve_length:%v", - "min_length:%v", - "max_length:%v", + "value:this.%s", + "preserve_length:%v", + "min_length:%v", + "max_length:%v", } params := []any{ - valuePath, - o.preserveLength, - o.minLength, - o.maxLength, + valuePath, + o.preserveLength, + o.minLength, + o.maxLength, } - template := fmt.Sprintf("transform_string(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_string(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go index 3124bd3856..d32a70dc4b 100644 --- a/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go +++ b/worker/pkg/benthos/transformers/gen_transform_string_phone_number.go @@ -30,12 +30,12 @@ func NewTransformStringPhoneNumberOpts( maxLengthArg *int64, seedArg *int64, ) (*TransformStringPhoneNumberOpts, error) { - preserveLength := bool(false) + preserveLength := bool(false) if preserveLengthArg != nil { preserveLength = *preserveLengthArg } - maxLength := int64(100) + maxLength := int64(100) if maxLengthArg != nil { maxLength = *maxLengthArg } @@ -56,18 +56,20 @@ func (o *TransformStringPhoneNumberOpts) BuildBloblangString( valuePath string, ) string { fnStr := []string{ - "value:this.%s", - "preserve_length:%v", - "max_length:%v", + "value:this.%s", + "preserve_length:%v", + "max_length:%v", } params := []any{ - valuePath, - o.preserveLength, - o.maxLength, + valuePath, + o.preserveLength, + o.maxLength, } - template := fmt.Sprintf("transform_phone_number(%s)", strings.Join(fnStr, ", ")) + + + template := fmt.Sprintf("transform_phone_number(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/neosync_transformer_generator.go b/worker/pkg/benthos/transformers/neosync_transformer_generator.go index 9e9d1de887..3454912da0 100644 --- a/worker/pkg/benthos/transformers/neosync_transformer_generator.go +++ b/worker/pkg/benthos/transformers/neosync_transformer_generator.go @@ -148,7 +148,11 @@ func New{{.StructName}}Opts( return nil, fmt.Errorf("unable to generate seed: %w", err) } {{ else if $param.HasDefault }} + {{- if eq $param.TypeStr "any" }} + var {{$param.Name}} any + {{- else}} {{$param.Name}} := {{$param.TypeStr}}({{$param.Default}}) + {{- end }} if {{$param.Name}}Arg != nil { {{$param.Name}} = *{{$param.Name}}Arg } @@ -175,9 +179,9 @@ func (o *{{.StructName}}Opts) BuildBloblangString( {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} {{- if eq $param.Name "value" }} - "value:this.%s", - {{- else }} - "{{$param.BloblangName}}:%v", + "value:this.%s", + {{- else if not $param.IsOptional }} + "{{$param.BloblangName}}:{{- if eq $param.TypeStr "string" }}%q{{else}}%v{{end}}", {{- end }} {{- end }} } @@ -186,24 +190,25 @@ func (o *{{.StructName}}Opts) BuildBloblangString( {{- range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} {{- if eq $param.Name "value" }} - valuePath, - {{- else }} - o.{{$param.Name}}, + valuePath, + {{- else if not $param.IsOptional }} + o.{{$param.Name}}, {{- end }} {{- end }} } - {{- range $index, $param := .FunctInfo.Params }} + {{ range $index, $param := .FunctInfo.Params }} {{- if eq $param.Name "value" }}{{ continue }}{{ end }} {{- if eq $param.Name "seed" }}{{ continue }}{{ end }} {{- if $param.IsOptional }} if o.{{$param.Name}} != nil { - fnStr = append(fnStr, "{{$param.BloblangName}}:%v") + fnStr = append(fnStr, "{{$param.BloblangName}}:{{- if eq $param.TypeStr "string" }}%q{{else}}%v{{end}}") + params = append(params, *o.{{$param.Name}}) } - {{- end }} + {{- end -}} {{- end }} - template := fmt.Sprintf("{{ .FunctInfo.BloblangFuncName }}(%s)", strings.Join(fnStr, ", ")) + template := fmt.Sprintf("{{ .FunctInfo.BloblangFuncName }}(%s)", strings.Join(fnStr, ",")) return fmt.Sprintf(template, params...) } diff --git a/worker/pkg/benthos/transformers/transform_email.go b/worker/pkg/benthos/transformers/transform_email.go index edb83a34a6..b6dfd77fae 100644 --- a/worker/pkg/benthos/transformers/transform_email.go +++ b/worker/pkg/benthos/transformers/transform_email.go @@ -1,6 +1,7 @@ package transformers import ( + "encoding/json" "errors" "fmt" "math" @@ -132,7 +133,8 @@ func init() { func NewTransformEmailOptsFromConfig(config *mgmtv1alpha1.TransformEmail, maxLength *int64) (*TransformEmailOpts, error) { if config == nil { - return NewTransformEmailOpts(nil, nil, nil, nil, nil, nil, nil) + var excludedDomains any = "[]" + return NewTransformEmailOpts(nil, nil, &excludedDomains, nil, nil, nil, nil) } var emailType *string if config.EmailType != nil { @@ -144,7 +146,11 @@ func NewTransformEmailOptsFromConfig(config *mgmtv1alpha1.TransformEmail, maxLen invalidEmailActionStr := dtoInvalidEmailActionToTransformerInvalidEmailAction(config.GetInvalidEmailAction()).String() invalidEmailAction = &invalidEmailActionStr } - var excludedDomains any = config.GetExcludedDomains() + excludedDomainsStr, err := convertStringSliceToString(config.GetExcludedDomains()) + if err != nil { + return nil, err + } + var excludedDomains any = excludedDomainsStr return NewTransformEmailOpts( config.PreserveLength, config.PreserveDomain, @@ -178,6 +184,9 @@ func (t *TransformEmail) Transform(value, opts any) (any, error) { excludedDomains = exDomainsStrs case []string: excludedDomains = v + case string: + css := strings.TrimSuffix(strings.TrimPrefix(v, "["), "]") + excludedDomains = strings.Split(css, ",") default: return nil, fmt.Errorf("excludedDomains is of type %T, not []any or []string", v) } @@ -345,3 +354,18 @@ func dtoInvalidEmailActionToTransformerInvalidEmailAction(dto mgmtv1alpha1.Inval return InvalidEmailAction_Reject } } + +func convertStringSliceToString(slc []string) (string, error) { + var returnStr string + + if len(slc) == 0 { + returnStr = "[]" + } else { + sliceBytes, err := json.Marshal(slc) + if err != nil { + return "", err + } + returnStr = string(sliceBytes) + } + return returnStr, nil +} diff --git a/worker/pkg/benthos/transformers/transform_email_test.go b/worker/pkg/benthos/transformers/transform_email_test.go index 8831f10fb7..0c53d4a0af 100644 --- a/worker/pkg/benthos/transformers/transform_email_test.go +++ b/worker/pkg/benthos/transformers/transform_email_test.go @@ -431,3 +431,19 @@ func Test_fromAnyToStringSlice(t *testing.T) { require.NoError(t, err) require.Empty(t, output) } + +func Test_ConverStringSliceToStringEmptySlice(t *testing.T) { + slc := []string{} + + res, err := convertStringSliceToString(slc) + require.NoError(t, err) + require.Equal(t, "[]", res) +} + +func Test_ConverStringSliceToStringNotEmptySlice(t *testing.T) { + slc := []string{"gmail.com", "yahoo.com"} + + res, err := convertStringSliceToString(slc) + require.NoError(t, err) + require.Equal(t, `["gmail.com","yahoo.com"]`, res) +} diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go index bd72dbf844..12e0125a6b 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go @@ -638,7 +638,7 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { TransformEmailConfig: &mgmtv1alpha1.TransformEmail{ PreserveDomain: gotypeutil.ToPtr(false), PreserveLength: gotypeutil.ToPtr(false), - ExcludedDomains: []string{}, + ExcludedDomains: []string{"gmail", "yahoo"}, EmailType: &uuidEmailType, }, }, @@ -1023,6 +1023,195 @@ func Test_computeMutationFunction_Validate_Bloblang_Output(t *testing.T) { } } +func Test_computeMutationFunction_Validate_Bloblang_Output_EmptyConfigs(t *testing.T) { + transformers := []*mgmtv1alpha1.SystemTransformer{ + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_EMAIL, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_EMAIL, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_BOOL, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CARD_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CITY, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_E164_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FIRST_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FLOAT64, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FULL_ADDRESS, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FULL_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_GENDER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_LAST_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_SHA256HASH, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_SSN, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STATE, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STREET_ADDRESS, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STRING_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_RANDOM_STRING, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UNIXTIMESTAMP, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_USERNAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UTCTIMESTAMP, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UUID, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_ZIPCODE, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_E164_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FIRST_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FLOAT64, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FULL_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_LAST_NAME, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_PHONE_NUMBER, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_STRING, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CATEGORICAL, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_CHARACTER_SCRAMBLE, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_DEFAULT, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_NULL, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + { + Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_COUNTRY, + Config: &mgmtv1alpha1.TransformerConfig{}, + }, + } + + emailColInfo := &sqlmanager_shared.ColumnInfo{ + OrdinalPosition: 2, + ColumnDefault: "", + IsNullable: true, + DataType: "timestamptz", + CharacterMaximumLength: shared.Ptr(40), + NumericPrecision: nil, + NumericScale: nil, + } + + for _, transformer := range transformers { + t.Run(fmt.Sprintf("%s_%s_lint", t.Name(), transformer.Source), func(t *testing.T) { + val, err := computeMutationFunction( + &mgmtv1alpha1.JobMapping{ + Column: "email", + Transformer: &mgmtv1alpha1.JobMappingTransformer{ + Source: transformer.Source, + Config: transformer.Config, + }, + }, emailColInfo, false) + require.NoError(t, err) + ex, err := bloblang.Parse(val) + require.NoError(t, err, fmt.Sprintf("transformer lint failed, check that the transformer string is being constructed correctly. Failing source: %s", transformer.Source)) + _, err = ex.Query(nil) + require.NoError(t, err) + }) + } +} + func Test_computeMutationFunction_handles_Db_Maxlen(t *testing.T) { type testcase struct { jm *mgmtv1alpha1.JobMapping @@ -1213,22 +1402,6 @@ func Test_buildBranchCacheConfigs_self_referencing(t *testing.T) { require.Len(t, resp, 0) } -func Test_ConverStringSliceToStringEmptySlice(t *testing.T) { - slc := []string{} - - res, err := convertStringSliceToString(slc) - require.NoError(t, err) - require.Equal(t, "[]", res) -} - -func Test_ConverStringSliceToStringNotEmptySlice(t *testing.T) { - slc := []string{"gmail.com", "yahoo.com"} - - res, err := convertStringSliceToString(slc) - require.NoError(t, err) - require.Equal(t, `["gmail.com","yahoo.com"]`, res) -} - func Test_getPrimaryKeyDependencyMap(t *testing.T) { tableDependencies := map[string][]*sqlmanager_shared.ForeignConstraint{ "hr.countries": { diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go index 527b098f16..24bfb22e03 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go @@ -432,67 +432,71 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s } formattedColPath := getBenthosColumnKey(col.Column, splitColumnPath) + config := col.GetTransformer().GetConfig() switch col.Transformer.Source { case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CATEGORICAL: - categories := col.Transformer.Config.GetGenerateCategoricalConfig().GetCategories() - return fmt.Sprintf(`generate_categorical(categories: %q)`, categories), nil - case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_EMAIL: - emailType := col.GetTransformer().GetConfig().GetGenerateEmailConfig().GetEmailType() - if emailType == mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_UNSPECIFIED { - emailType = mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_UUID_V4 - } - return fmt.Sprintf(`generate_email(max_length:%d,email_type:%q)`, maxLen, dtoEmailTypeToBenthosEmailType(emailType)), nil - case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_EMAIL: - pd := col.Transformer.Config.GetTransformEmailConfig().GetPreserveDomain() - pl := col.Transformer.Config.GetTransformEmailConfig().GetPreserveLength() - excludedDomains := col.Transformer.Config.GetTransformEmailConfig().GetExcludedDomains() - - excludedDomainsStr, err := convertStringSliceToString(excludedDomains) + opts, err := transformers.NewGenerateCategoricalOptsFromConfig(config.GetGenerateCategoricalConfig()) if err != nil { return "", err } - emailType := col.GetTransformer().GetConfig().GetTransformEmailConfig().GetEmailType() - if emailType == mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_UNSPECIFIED { - emailType = mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_UUID_V4 + return opts.BuildBloblangString(), nil + case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_EMAIL: + opts, err := transformers.NewGenerateEmailOptsFromConfig(config.GetGenerateEmailConfig(), &maxLen) + if err != nil { + return "", err } - - invalidEmailAction := col.GetTransformer().GetConfig().GetTransformEmailConfig().GetInvalidEmailAction() - if invalidEmailAction == mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_UNSPECIFIED { - invalidEmailAction = mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_REJECT + return opts.BuildBloblangString(), nil + case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_EMAIL: + opts, err := transformers.NewTransformEmailOptsFromConfig(config.GetTransformEmailConfig(), &maxLen) + if err != nil { + return "", nil } - - return fmt.Sprintf( - "transform_email(value:this.%s,preserve_domain:%t,preserve_length:%t,excluded_domains:%v,max_length:%d,email_type:%q,invalid_email_action:%q)", - formattedColPath, pd, pl, excludedDomainsStr, maxLen, dtoEmailTypeToBenthosEmailType(emailType), dtoInvalidEmailActionToBenthosInvalidEmailAction(invalidEmailAction), - ), nil + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_BOOL: - return "generate_bool()", nil + opts, err := transformers.NewGenerateBoolOptsFromConfig(config.GetGenerateBoolConfig()) + if err != nil { + return "", nil + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CARD_NUMBER: - luhn := col.Transformer.Config.GetGenerateCardNumberConfig().GetValidLuhn() - return fmt.Sprintf(`generate_card_number(valid_luhn:%t)`, luhn), nil + opts, err := transformers.NewGenerateCardNumberOptsFromConfig(config.GetGenerateCardNumberConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_CITY: - return fmt.Sprintf(`generate_city(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateCityOptsFromConfig(config.GetGenerateCityConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_E164_PHONE_NUMBER: - minValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().GetMin() - maxValue := col.Transformer.Config.GetGenerateE164PhoneNumberConfig().GetMax() - return fmt.Sprintf(`generate_e164_phone_number(min:%d,max:%d)`, minValue, maxValue), nil + opts, err := transformers.NewGenerateInternationalPhoneNumberOptsFromConfig(config.GetGenerateE164PhoneNumberConfig()) + if err != nil { + return "", nil + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FIRST_NAME: - return fmt.Sprintf(`generate_first_name(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateFirstNameOptsFromConfig(config.GetGenerateFirstNameConfig(), &maxLen) + if err != nil { + return "", nil + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FLOAT64: - randomSign := col.Transformer.Config.GetGenerateFloat64Config().GetRandomizeSign() - minValue := col.Transformer.Config.GetGenerateFloat64Config().GetMin() - maxValue := col.Transformer.Config.GetGenerateFloat64Config().GetMax() - var precision *int64 - if col.GetTransformer().GetConfig().GetGenerateFloat64Config().GetPrecision() > 0 { - userDefinedPrecision := col.GetTransformer().GetConfig().GetGenerateFloat64Config().GetPrecision() + if config != nil && config.GetGenerateFloat64Config() != nil && config.GetGenerateFloat64Config().GetPrecision() > 0 { + userDefinedPrecision := config.GetGenerateFloat64Config().GetPrecision() precision = &userDefinedPrecision + config.GetGenerateFloat64Config().Precision = &userDefinedPrecision } if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { newPrecision := transformer_utils.Ceil(*precision, int64(*colInfo.NumericPrecision)) precision = &newPrecision } + if config != nil && config.GetGenerateFloat64Config() != nil && precision != nil { + config.GetGenerateFloat64Config().Precision = precision + } var scale *int64 if colInfo != nil && colInfo.NumericScale != nil && *colInfo.NumericScale >= 0 { @@ -500,81 +504,132 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s scale = &newScale } - fnStr := []string{"randomize_sign:%t", "min:%f", "max:%f"} - params := []any{randomSign, minValue, maxValue} - - if precision != nil { - fnStr = append(fnStr, "precision: %d") - params = append(params, *precision) - } - if scale != nil { - fnStr = append(fnStr, "scale: %d") - params = append(params, *scale) + opts, err := transformers.NewGenerateFloat64OptsFromConfig(col.Transformer.Config.GetGenerateFloat64Config(), scale) + if err != nil { + return "", err } - template := fmt.Sprintf("generate_float64(%s)", strings.Join(fnStr, ", ")) - return fmt.Sprintf(template, params...), nil + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FULL_ADDRESS: - return fmt.Sprintf(`generate_full_address(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateFullAddressOptsFromConfig(config.GetGenerateFullAddressConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_FULL_NAME: - return fmt.Sprintf(`generate_full_name(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateFullNameOptsFromConfig(config.GetGenerateFullNameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_GENDER: - ab := col.Transformer.Config.GetGenerateGenderConfig().GetAbbreviate() - return fmt.Sprintf(`generate_gender(abbreviate:%t,max_length:%d)`, ab, maxLen), nil + opts, err := transformers.NewGenerateGenderOptsFromConfig(config.GetGenerateGenderConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64_PHONE_NUMBER: - return "generate_int64_phone_number()", nil + opts, err := transformers.NewGenerateInt64PhoneNumberOptsFromConfig(config.GetGenerateInt64PhoneNumberConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_INT64: - sign := col.Transformer.Config.GetGenerateInt64Config().GetRandomizeSign() - minValue := col.Transformer.Config.GetGenerateInt64Config().GetMin() - maxValue := col.Transformer.Config.GetGenerateInt64Config().GetMax() - return fmt.Sprintf(`generate_int64(randomize_sign:%t,min:%d, max:%d)`, sign, minValue, maxValue), nil + opts, err := transformers.NewGenerateInt64OptsFromConfig(config.GetGenerateInt64Config()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_LAST_NAME: - return fmt.Sprintf(`generate_last_name(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateLastNameOptsFromConfig(config.GetGenerateLastNameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_SHA256HASH: - return `generate_sha256hash()`, nil + opts, err := transformers.NewGenerateSHA256HashOptsFromConfig(config.GetGenerateSha256HashConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_SSN: - return "generate_ssn()", nil + opts, err := transformers.NewGenerateSSNOptsFromConfig(config.GetGenerateSsnConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STATE: - generateFullName := col.Transformer.Config.GetGenerateStateConfig().GetGenerateFullName() - return fmt.Sprintf(`generate_state(generate_full_name:%t)`, generateFullName), nil + opts, err := transformers.NewGenerateStateOptsFromConfig(config.GetGenerateStateConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STREET_ADDRESS: - return fmt.Sprintf(`generate_street_address(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateStreetAddressOptsFromConfig(config.GetGenerateStreetAddressConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_STRING_PHONE_NUMBER: - minValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().GetMin() - maxValue := col.Transformer.Config.GetGenerateStringPhoneNumberConfig().GetMax() - minValue = transformer_utils.MinInt(minValue, maxLen) - maxValue = transformer_utils.Ceil(maxValue, maxLen) - return fmt.Sprintf("generate_string_phone_number(min:%d,max:%d)", minValue, maxValue), nil + opts, err := transformers.NewGenerateStringPhoneNumberOptsFromConfig(config.GetGenerateStringPhoneNumberConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_RANDOM_STRING: - minValue := col.Transformer.Config.GetGenerateStringConfig().GetMin() - maxValue := col.Transformer.Config.GetGenerateStringConfig().GetMax() - minValue = transformer_utils.MinInt(minValue, maxLen) // ensure the min is not larger than the max allowed length - maxValue = transformer_utils.Ceil(maxValue, maxLen) // todo: we need to pull in the min from the database schema - return fmt.Sprintf(`generate_string(min:%d,max:%d)`, minValue, maxValue), nil + opts, err := transformers.NewGenerateRandomStringOptsFromConfig(config.GetGenerateStringConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UNIXTIMESTAMP: - return "generate_unixtimestamp()", nil + opts, err := transformers.NewGenerateUnixTimestampOptsFromConfig(config.GetGenerateUnixtimestampConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_USERNAME: - return fmt.Sprintf(`generate_username(max_length:%d)`, maxLen), nil + opts, err := transformers.NewGenerateUsernameOptsFromConfig(config.GetGenerateUsernameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UTCTIMESTAMP: - return "generate_utctimestamp()", nil + opts, err := transformers.NewGenerateUTCTimestampOptsFromConfig(config.GetGenerateUtctimestampConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_UUID: - ih := col.Transformer.Config.GetGenerateUuidConfig().GetIncludeHyphens() - return fmt.Sprintf("generate_uuid(include_hyphens:%t)", ih), nil + opts, err := transformers.NewGenerateUUIDOptsFromConfig(config.GetGenerateUuidConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_ZIPCODE: - return "generate_zipcode()", nil + opts, err := transformers.NewGenerateZipcodeOptsFromConfig(config.GetGenerateZipcodeConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_E164_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformE164PhoneNumberConfig().GetPreserveLength() - return fmt.Sprintf("transform_e164_phone_number(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil + opts, err := transformers.NewTransformE164PhoneNumberOptsFromConfig(config.GetTransformE164PhoneNumberConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FIRST_NAME: - pl := col.Transformer.Config.GetTransformFirstNameConfig().GetPreserveLength() - return fmt.Sprintf("transform_first_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil + opts, err := transformers.NewTransformFirstNameOptsFromConfig(config.GetTransformFirstNameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FLOAT64: var precision *int64 if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { newPrecision := int64(*colInfo.NumericPrecision) precision = &newPrecision } - var scale *int64 if colInfo != nil && colInfo.NumericScale != nil && *colInfo.NumericScale >= 0 { newScale := int64(*colInfo.NumericScale) @@ -584,110 +639,61 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s if err != nil { return "", err } - return opts.BuildBenthosString(formattedColPath), nil - // config := col.Transformer.Config.GetTransformFloat64Config() - // var randomMin *float64 - // if config != nil && config.RandomizationRangeMin != nil { - // randomMin = config.RandomizationRangeMin - // } - // var randomMax *float64 - // if config != nil && config.RandomizationRangeMax != nil { - // randomMax = config.RandomizationRangeMax - // } - - // var precision *int64 - // if colInfo != nil && colInfo.NumericPrecision != nil && *colInfo.NumericPrecision > 0 { - // newPrecision := int64(*colInfo.NumericPrecision) - // precision = &newPrecision - // } - - // var scale *int64 - // if colInfo != nil && colInfo.NumericScale != nil && *colInfo.NumericScale >= 0 { - // newScale := int64(*colInfo.NumericScale) - // scale = &newScale - // } - - // fnStr := []string{"value:this.%s"} - // params := []any{formattedColPath} - - // if precision != nil { - // fnStr = append(fnStr, "precision:%d") - // params = append(params, *precision) - // } - // if scale != nil { - // fnStr = append(fnStr, "scale:%d") - // params = append(params, *scale) - // } - // if randomMin != nil { - // fnStr = append(fnStr, "randomization_range_min:%f") - // params = append(params, *randomMin) - // } - // if randomMax != nil { - // fnStr = append(fnStr, "randomization_range_max:%f") - // params = append(params, *randomMax) - // } - // template := fmt.Sprintf(`transform_float64(%s)`, strings.Join(fnStr, ", ")) - // return fmt.Sprintf(template, params...), nil + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_FULL_NAME: - pl := col.Transformer.Config.GetTransformFullNameConfig().GetPreserveLength() - return fmt.Sprintf("transform_full_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil + opts, err := transformers.NewTransformFullNameOptsFromConfig(config.GetTransformFullNameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformInt64PhoneNumberConfig().GetPreserveLength() - return fmt.Sprintf("transform_int64_phone_number(value:this.%s,preserve_length:%t)", formattedColPath, pl), nil + opts, err := transformers.NewTransformInt64PhoneNumberOptsFromConfig(config.GetTransformInt64PhoneNumberConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_INT64: - rMin := col.Transformer.Config.GetTransformInt64Config().GetRandomizationRangeMin() - rMax := col.Transformer.Config.GetTransformInt64Config().GetRandomizationRangeMax() - return fmt.Sprintf(`transform_int64(value:this.%s,randomization_range_min:%d,randomization_range_max:%d)`, formattedColPath, rMin, rMax), nil + opts, err := transformers.NewTransformInt64OptsFromConfig(config.GetTransformInt64Config()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_LAST_NAME: - pl := col.Transformer.Config.GetTransformLastNameConfig().GetPreserveLength() - return fmt.Sprintf("transform_last_name(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil + opts, err := transformers.NewTransformLastNameOptsFromConfig(config.GetTransformLastNameConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_PHONE_NUMBER: - pl := col.Transformer.Config.GetTransformPhoneNumberConfig().GetPreserveLength() - return fmt.Sprintf("transform_phone_number(value:this.%s,preserve_length:%t,max_length:%d)", formattedColPath, pl, maxLen), nil + opts, err := transformers.NewTransformStringPhoneNumberOptsFromConfig(config.GetTransformPhoneNumberConfig(), &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_STRING: - pl := col.Transformer.Config.GetTransformStringConfig().GetPreserveLength() minLength := int64(3) // todo: we need to pull in this value from the database schema - return fmt.Sprintf(`transform_string(value:this.%s,preserve_length:%t,min_length:%d,max_length:%d)`, formattedColPath, pl, minLength, maxLen), nil + opts, err := transformers.NewTransformStringOptsFromConfig(config.GetTransformStringConfig(), &minLength, &maxLen) + if err != nil { + return "", err + } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_NULL: return shared.NullString, nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_DEFAULT: return `"DEFAULT"`, nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_CHARACTER_SCRAMBLE: - regex := col.Transformer.Config.GetTransformCharacterScrambleConfig().UserProvidedRegex - - if regex != nil { - regexValue := *regex - return fmt.Sprintf(`transform_character_scramble(value:this.%s,user_provided_regex:%q)`, formattedColPath, regexValue), nil - } else { - return fmt.Sprintf(`transform_character_scramble(value:this.%s)`, formattedColPath), nil + opts, err := transformers.NewTransformCharacterScrambleOptsFromConfig(config.GetTransformCharacterScrambleConfig()) + if err != nil { + return "", err } + return opts.BuildBloblangString(formattedColPath), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_COUNTRY: - generateFullName := col.Transformer.Config.GetGenerateCountryConfig().GetGenerateFullName() - return fmt.Sprintf(`generate_country(generate_full_name:%t)`, generateFullName), nil - + opts, err := transformers.NewGenerateCountryOptsFromConfig(config.GetGenerateCountryConfig()) + if err != nil { + return "", err + } + return opts.BuildBloblangString(), nil default: return "", fmt.Errorf("unsupported transformer") } } - -func dtoEmailTypeToBenthosEmailType(dto mgmtv1alpha1.GenerateEmailType) transformers.GenerateEmailType { - switch dto { - case mgmtv1alpha1.GenerateEmailType_GENERATE_EMAIL_TYPE_FULLNAME: - return transformers.GenerateEmailType_FullName - default: - return transformers.GenerateEmailType_UuidV4 - } -} - -func dtoInvalidEmailActionToBenthosInvalidEmailAction(dto mgmtv1alpha1.InvalidEmailAction) transformers.InvalidEmailAction { - switch dto { - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_GENERATE: - return transformers.InvalidEmailAction_Generate - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_NULL: - return transformers.InvalidEmailAction_Null - case mgmtv1alpha1.InvalidEmailAction_INVALID_EMAIL_ACTION_PASSTHROUGH: - return transformers.InvalidEmailAction_Passthrough - default: - return transformers.InvalidEmailAction_Reject - } -} diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/utils.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/utils.go index 620c787367..5da2ccc3ac 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/utils.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/utils.go @@ -1,28 +1,12 @@ package genbenthosconfigs_activity import ( - "encoding/json" "fmt" "strings" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" ) -func convertStringSliceToString(slc []string) (string, error) { - var returnStr string - - if len(slc) == 0 { - returnStr = "[]" - } else { - sliceBytes, err := json.Marshal(slc) - if err != nil { - return "", err - } - returnStr = string(sliceBytes) - } - return returnStr, nil -} - func getMapValuesCount[K comparable, V any](m map[K][]V) int { count := 0 for _, v := range m { From 1d4658ce8b9c91228493e3f964a8aab27eccd25a Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 15:59:20 -0700 Subject: [PATCH 29/32] fix test --- .../activities/gen-benthos-configs/benthos-builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go index 12e0125a6b..63bc7f1a53 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/benthos-builder_test.go @@ -400,7 +400,7 @@ func Test_buildProcessorConfigsMutation(t *testing.T) { {Schema: "public", Table: "users", Column: "email", Transformer: &mgmtv1alpha1.JobMappingTransformer{Source: jsT.Source, Config: jsT.Config}}}, groupedSchemas, map[string][]*referenceKey{}, []string{}, mockJobId, mockRunId, nil, runconfig, nil, []string{}) require.Nil(t, err) - require.Equal(t, *output[0].Mutation, `root."email" = transform_email(value:this."email",preserve_domain:true,preserve_length:false,excluded_domains:[],max_length:40,email_type:"uuidv4",invalid_email_action:"reject")`) + require.Equal(t, `root."email" = transform_email(value:this."email",preserve_length:false,preserve_domain:true,excluded_domains:[],max_length:40,email_type:"uuidv4",invalid_email_action:"reject")`, *output[0].Mutation) } func Test_ShouldProcessColumnTrue(t *testing.T) { From 989b296aaad014cd21ff9b21f14f3b9789100f9e Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 17:01:39 -0700 Subject: [PATCH 30/32] fix test --- .../transformers/generate_random_string.go | 15 +++++++++++---- .../transformers/transformer_initializer.go | 2 +- .../transformers/transformer_initializer_test.go | 1 + .../activities/gen-benthos-configs/processors.go | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_random_string.go index 16bd89643b..e3f4297c22 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -55,17 +55,24 @@ func init() { } } -func NewGenerateRandomStringOptsFromConfig(config *mgmtv1alpha1.GenerateString) (*GenerateRandomStringOpts, error) { +func NewGenerateRandomStringOptsFromConfig(config *mgmtv1alpha1.GenerateString, maxLen *int64) (*GenerateRandomStringOpts, error) { if config == nil { return NewGenerateRandomStringOpts( nil, - nil, + maxLen, nil, ) } + minValue := config.GetMin() + maxValue := config.GetMax() + if maxLen != nil { + minValue = transformer_utils.MinInt(minValue, *maxLen) // ensure the min is not larger than the max allowed length + maxValue = transformer_utils.Ceil(maxValue, *maxLen) + } + return NewGenerateRandomStringOpts( - config.Min, - config.Max, + &minValue, + &maxValue, nil, ) } diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformers/transformer_initializer.go index 28b42345c6..e6999a3404 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformers/transformer_initializer.go @@ -377,7 +377,7 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOptsFromConfig(config) + opts, err := NewGenerateRandomStringOptsFromConfig(config, &maxLength) if err != nil { return nil, err } diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformers/transformer_initializer_test.go index e93c891804..153fc346fe 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer_test.go +++ b/worker/pkg/benthos/transformers/transformer_initializer_test.go @@ -275,6 +275,7 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { PreserveLength: &preserve, EmailType: &emailType, InvalidEmailAction: &invalidEmailAction, + ExcludedDomains: []string{"gmail", "yahoo"}, }, }, } diff --git a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go index 24bfb22e03..7b3667ad69 100644 --- a/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go +++ b/worker/pkg/workflows/datasync/activities/gen-benthos-configs/processors.go @@ -577,7 +577,7 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_s return opts.BuildBloblangString(), nil case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_RANDOM_STRING: // todo: we need to pull in the min from the database schema - opts, err := transformers.NewGenerateRandomStringOptsFromConfig(config.GetGenerateStringConfig()) + opts, err := transformers.NewGenerateRandomStringOptsFromConfig(config.GetGenerateStringConfig(), &maxLen) if err != nil { return "", err } From d9ff4901b627099753ca73ed84f29d1ba9ca4ba0 Mon Sep 17 00:00:00 2001 From: Alisha Date: Fri, 18 Oct 2024 17:15:54 -0700 Subject: [PATCH 31/32] fix random string bounds --- .../transformers/generate_random_string.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_random_string.go index e3f4297c22..57ea979e83 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -63,16 +63,20 @@ func NewGenerateRandomStringOptsFromConfig(config *mgmtv1alpha1.GenerateString, nil, ) } - minValue := config.GetMin() - maxValue := config.GetMax() - if maxLen != nil { - minValue = transformer_utils.MinInt(minValue, *maxLen) // ensure the min is not larger than the max allowed length - maxValue = transformer_utils.Ceil(maxValue, *maxLen) + minValue := config.Min + maxValue := config.Max + if maxLen != nil && maxValue != nil { + newMax := transformer_utils.Ceil(*maxValue, *maxLen) + maxValue = &newMax + } + if minValue != nil { + newMin := transformer_utils.MinInt(*minValue, *maxValue) // ensure the min is not larger than the max allowed length + minValue = &newMin } return NewGenerateRandomStringOpts( - &minValue, - &maxValue, + minValue, + maxValue, nil, ) } From 3bdd54234ac3c199b536636a684be4bdc58d74cf Mon Sep 17 00:00:00 2001 From: Alisha Date: Mon, 21 Oct 2024 10:56:19 -0700 Subject: [PATCH 32/32] fix random string bounds and add source file to transformer generator log --- .../pkg/benthos/transformers/generate_random_string.go | 10 +++++++--- worker/pkg/benthos/transformers/generator_utils.go | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/worker/pkg/benthos/transformers/generate_random_string.go b/worker/pkg/benthos/transformers/generate_random_string.go index 57ea979e83..4f0a4cca0b 100644 --- a/worker/pkg/benthos/transformers/generate_random_string.go +++ b/worker/pkg/benthos/transformers/generate_random_string.go @@ -65,9 +65,13 @@ func NewGenerateRandomStringOptsFromConfig(config *mgmtv1alpha1.GenerateString, } minValue := config.Min maxValue := config.Max - if maxLen != nil && maxValue != nil { - newMax := transformer_utils.Ceil(*maxValue, *maxLen) - maxValue = &newMax + if maxLen != nil { + if maxValue == nil { + maxValue = maxLen + } else { + newMax := transformer_utils.Ceil(*maxValue, *maxLen) + maxValue = &newMax + } } if minValue != nil { newMin := transformer_utils.MinInt(*minValue, *maxValue) // ensure the min is not larger than the max allowed length diff --git a/worker/pkg/benthos/transformers/generator_utils.go b/worker/pkg/benthos/transformers/generator_utils.go index a0441d6732..0960af1035 100644 --- a/worker/pkg/benthos/transformers/generator_utils.go +++ b/worker/pkg/benthos/transformers/generator_utils.go @@ -156,7 +156,7 @@ func ParseBloblangSpec(benthosSpec *BenthosSpec) (*ParsedBenthosSpec, error) { } } - bloblangFuncName, err := extractBloblangFunctionName(benthosSpecStr) + bloblangFuncName, err := extractBloblangFunctionName(benthosSpecStr, benthosSpec.SourceFile) if err != nil { return nil, err } @@ -168,7 +168,7 @@ func ParseBloblangSpec(benthosSpec *BenthosSpec) (*ParsedBenthosSpec, error) { }, nil } -func extractBloblangFunctionName(input string) (string, error) { +func extractBloblangFunctionName(input, sourceFile string) (string, error) { // Looks for bloblang.RegisterFunctionV2 and captures the function name in quotes re := regexp.MustCompile(`bloblang\.RegisterFunctionV2\("([^"]+)"`) @@ -178,7 +178,7 @@ func extractBloblangFunctionName(input string) (string, error) { return matches[1], nil } - return "", fmt.Errorf("bloblang function name not found") + return "", fmt.Errorf("bloblang function name not found: %s", sourceFile) } func lowercaseFirst(s string) string {