Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Makes all transformer arugments optional and adds tests #2817

Merged
merged 35 commits into from
Oct 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
7177207
update generators to have optional params
alishakawaguchi Oct 15, 2024
c742a88
Merge branch 'main' into alisha/transformer-clean-up
alishakawaguchi Oct 15, 2024
0d4a46c
make transformer arguments optional
alishakawaguchi Oct 15, 2024
2ff876e
add transfomer initializer tests
alishakawaguchi Oct 16, 2024
3da7202
change generate_random_string to match proto
alishakawaguchi Oct 16, 2024
20d5ef0
change generate_e164_phone_number to match proto
alishakawaguchi Oct 16, 2024
78d92f4
change transform_string_phone_number to match proto
alishakawaguchi Oct 16, 2024
788dad2
fix lint
alishakawaguchi Oct 16, 2024
a9a32c9
fix javascript integration test
alishakawaguchi Oct 16, 2024
6da5708
revert breaking changes to transformer names
alishakawaguchi Oct 16, 2024
e08437b
make transformer config args optional in proto
alishakawaguchi Oct 17, 2024
9e9784f
stash
alishakawaguchi Oct 17, 2024
882c60a
stash
alishakawaguchi Oct 17, 2024
066c887
stash
alishakawaguchi Oct 17, 2024
dc40cb9
fixed transformer initializer
alishakawaguchi Oct 17, 2024
5367ccd
make transformer config args optional in model
alishakawaguchi Oct 17, 2024
9cf030d
fix ptr types
alishakawaguchi Oct 17, 2024
fd57798
revert generator changes
alishakawaguchi Oct 17, 2024
d11359d
remove unused code
alishakawaguchi Oct 17, 2024
4b8df8c
fix spelling
alishakawaguchi Oct 17, 2024
dccb9fe
Merge branch 'main' into alisha/transformer-clean-up
alishakawaguchi Oct 18, 2024
60bc95f
add build bloblang string to transformer generator
alishakawaguchi Oct 18, 2024
dfb3c69
forgot file
alishakawaguchi Oct 18, 2024
d0c97cc
update generator to get bloblang function name
alishakawaguchi Oct 18, 2024
0c65191
ok
alishakawaguchi Oct 18, 2024
e90007a
generator working
alishakawaguchi Oct 18, 2024
05d7bcd
fix generator import
alishakawaguchi Oct 18, 2024
3c3b3ab
fix generator import
alishakawaguchi Oct 18, 2024
795c11a
update gen files to have bloblang function
alishakawaguchi Oct 18, 2024
49533bb
update transformer processors
alishakawaguchi Oct 18, 2024
1d4658c
fix test
alishakawaguchi Oct 18, 2024
989b296
fix test
alishakawaguchi Oct 19, 2024
d9ff490
fix random string bounds
alishakawaguchi Oct 19, 2024
3bdd542
fix random string bounds and add source file to transformer generator…
alishakawaguchi Oct 21, 2024
b4c4bec
Merge branch 'main' into alisha/transformer-clean-up
alishakawaguchi Oct 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,101 changes: 582 additions & 519 deletions backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go

Large diffs are not rendered by default.

64 changes: 32 additions & 32 deletions backend/protos/mgmt/v1alpha1/transformer.proto
Original file line number Diff line number Diff line change
Expand Up @@ -317,9 +317,9 @@

message TransformEmail {
// Whether or not to preserve the original domain, barring what has been specified in the excluded_domains property.
bool preserve_domain = 1;
optional bool preserve_domain = 1;

Check failure on line 320 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "1" with name "preserve_domain" on message "TransformEmail" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
// Whether or not to preserve the original length of the email. This causes the transformed email to retain the original length.
bool preserve_length = 2;
optional bool preserve_length = 2;

Check failure on line 322 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "2" with name "preserve_length" on message "TransformEmail" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
// A lsit of email domains that should be excluded. This changes based on the preserve_domain flag. See the docs for more details.
repeated string excluded_domains = 3;
// Optionally specify the type of email to generate. The types specified determine the contents on the left side of the @.
Expand All @@ -331,41 +331,41 @@
message GenerateBool {}

message GenerateCardNumber {
bool valid_luhn = 1;
optional bool valid_luhn = 1;

Check failure on line 334 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "1" with name "valid_luhn" on message "GenerateCardNumber" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
}

message GenerateCity {}

message GenerateDefault {}

message GenerateE164PhoneNumber {
int64 min = 1;
int64 max = 2;
optional int64 min = 1;

Check failure on line 342 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "1" with name "min" on message "GenerateE164PhoneNumber" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
optional int64 max = 2;

Check failure on line 343 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "2" with name "max" on message "GenerateE164PhoneNumber" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
}

message GenerateFirstName {}

message GenerateFloat64 {
bool randomize_sign = 1;
double min = 2;
double max = 3;
int64 precision = 4;
optional bool randomize_sign = 1;

Check failure on line 349 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "1" with name "randomize_sign" on message "GenerateFloat64" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
optional double min = 2;

Check failure on line 350 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "2" with name "min" on message "GenerateFloat64" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
optional double max = 3;

Check failure on line 351 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "3" with name "max" on message "GenerateFloat64" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
optional int64 precision = 4;

Check failure on line 352 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "4" with name "precision" on message "GenerateFloat64" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
}

message GenerateFullAddress {}

message GenerateFullName {}

message GenerateGender {
bool abbreviate = 1;
optional bool abbreviate = 1;

Check failure on line 360 in backend/protos/mgmt/v1alpha1/transformer.proto

View workflow job for this annotation

GitHub Actions / buf

Field "1" with name "abbreviate" on message "GenerateGender" changed cardinality from "optional with implicit presence" to "optional with explicit presence".
}

message GenerateInt64PhoneNumber {}

message GenerateInt64 {
bool randomize_sign = 1;
int64 min = 2;
int64 max = 3;
optional bool randomize_sign = 1;
optional int64 min = 2;
optional int64 max = 3;
}

message GenerateLastName {}
Expand All @@ -375,19 +375,19 @@

message GenerateState {
// An option to return the full state name of the randomly selected state or return the default of a 2-letter state code.
bool generate_full_name = 1;
optional bool generate_full_name = 1;
}

message GenerateStreetAddress {}

message GenerateStringPhoneNumber {
int64 min = 2;
int64 max = 3;
optional int64 min = 2;
optional int64 max = 3;
}

message GenerateString {
int64 min = 1;
int64 max = 2;
optional int64 min = 1;
optional int64 max = 2;
}

message GenerateUnixTimestamp {}
Expand All @@ -397,47 +397,47 @@
message GenerateUtcTimestamp {}

message GenerateUuid {
bool include_hyphens = 1;
optional bool include_hyphens = 1;
}

message GenerateZipcode {}

message TransformE164PhoneNumber {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformFirstName {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformFloat64 {
double randomization_range_min = 1;
double randomization_range_max = 2;
optional double randomization_range_min = 1;
optional double randomization_range_max = 2;
}

message TransformFullName {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformInt64PhoneNumber {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformInt64 {
int64 randomization_range_min = 1;
int64 randomization_range_max = 2;
optional int64 randomization_range_min = 1;
optional int64 randomization_range_max = 2;
}

message TransformLastName {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformPhoneNumber {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message TransformString {
bool preserve_length = 1;
optional bool preserve_length = 1;
}

message Passthrough {}
Expand All @@ -463,7 +463,7 @@
}

message GenerateCategorical {
string categories = 1;
optional string categories = 1;
}

message TransformCharacterScramble {
Expand All @@ -487,7 +487,7 @@

message GenerateCountry {
// An option to return the full country name of the randomly selected country or return the default of a 2-letter country code.
bool generate_full_name = 1;
optional bool generate_full_name = 1;
}

service TransformersService {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"connectrpc.com/connect"
mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1"
"github.com/nucleuscloud/neosync/internal/gotypeutil"
"github.com/stretchr/testify/require"
"github.com/stripe/stripe-go/v79"
)
Expand Down Expand Up @@ -105,8 +106,8 @@ func (s *IntegrationTestSuite) Test_AnonymizeService_AnonymizeMany() {
N: &mgmtv1alpha1.TransformerConfig{
Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{
GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{
Min: 18,
Max: 30,
Min: gotypeutil.ToPtr(int64(18)),
Max: gotypeutil.ToPtr(int64(30)),
},
},
},
Expand Down Expand Up @@ -179,8 +180,8 @@ func (s *IntegrationTestSuite) Test_AnonymizeService_AnonymizeSingle() {
N: &mgmtv1alpha1.TransformerConfig{
Config: &mgmtv1alpha1.TransformerConfig_GenerateInt64Config{
GenerateInt64Config: &mgmtv1alpha1.GenerateInt64{
Min: 18,
Max: 30,
Min: gotypeutil.ToPtr(int64(18)),
Max: gotypeutil.ToPtr(int64(30)),
},
},
},
Expand Down
Loading