From b0f0565f7291e549002a0391cc5504bdad57437d Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 14 Nov 2023 13:56:52 -0800 Subject: [PATCH 1/5] updated card number transfomer --- .../go/protos/mgmt/v1alpha1/transformer.pb.go | 50 ++++++++-------- .../mgmt/v1alpha1/transformer.pb.validate.go | 60 +++++++++---------- .../protos/mgmt/v1alpha1/transformer.proto | 4 +- .../transformers-service/transformers.go | 8 +-- backend/sql/postgresql/models/transformers.go | 21 ++++--- docs/docs/transformers/system/card-number.mdx | 4 +- ...sx => CustomCardNumberTransformerForm.tsx} | 4 +- .../HandleCustomTransformersForm.tsx | 6 +- frontend/app/new/transformer/schema.ts | 8 ++- .../transformers/EditTransformerOptions.tsx | 12 ++-- ...Form.tsx => CardNumberTransformerForm.tsx} | 8 +-- .../mgmt/v1alpha1/transformer_pb.ts | 32 +++++----- frontend/yup-validations/transformers.ts | 16 ++--- .../{credit_card.go => card_number.go} | 16 ++--- ...redit_card_test.go => card_number_test.go} | 24 ++++---- .../datasync/activities/activities.go | 6 +- 16 files changed, 142 insertions(+), 137 deletions(-) rename frontend/app/new/transformer/CustomTransformerForms/{CustomCreditCardTransformerForm.tsx => CustomCardNumberTransformerForm.tsx} (89%) rename frontend/app/transformers/Sheetforms/{CreditCardTransformerForm.tsx => CardNumberTransformerForm.tsx} (89%) rename worker/internal/benthos/transformers/{credit_card.go => card_number.go} (81%) rename worker/internal/benthos/transformers/{credit_card_test.go => card_number_test.go} (52%) diff --git a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go index aaa7076d7b..d0a37f3da7 100644 --- a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go +++ b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.go @@ -928,7 +928,7 @@ type TransformerConfig struct { // *TransformerConfig_ZipcodeConfig // *TransformerConfig_StateConfig // *TransformerConfig_FullAddressConfig - // *TransformerConfig_CreditCardConfig + // *TransformerConfig_CardNumberConfig // *TransformerConfig_Sha256HashConfig // *TransformerConfig_SsnConfig Config isTransformerConfig_Config `protobuf_oneof:"config"` @@ -1120,9 +1120,9 @@ func (x *TransformerConfig) GetFullAddressConfig() *FullAddress { return nil } -func (x *TransformerConfig) GetCreditCardConfig() *CreditCard { - if x, ok := x.GetConfig().(*TransformerConfig_CreditCardConfig); ok { - return x.CreditCardConfig +func (x *TransformerConfig) GetCardNumberConfig() *CardNumber { + if x, ok := x.GetConfig().(*TransformerConfig_CardNumberConfig); ok { + return x.CardNumberConfig } return nil } @@ -1229,8 +1229,8 @@ type TransformerConfig_FullAddressConfig struct { FullAddressConfig *FullAddress `protobuf:"bytes,21,opt,name=full_address_config,json=fullAddressConfig,proto3,oneof"` } -type TransformerConfig_CreditCardConfig struct { - CreditCardConfig *CreditCard `protobuf:"bytes,22,opt,name=credit_card_config,json=creditCardConfig,proto3,oneof"` +type TransformerConfig_CardNumberConfig struct { + CardNumberConfig *CardNumber `protobuf:"bytes,22,opt,name=card_number_config,json=cardNumberConfig,proto3,oneof"` } type TransformerConfig_Sha256HashConfig struct { @@ -1283,7 +1283,7 @@ func (*TransformerConfig_StateConfig) isTransformerConfig_Config() {} func (*TransformerConfig_FullAddressConfig) isTransformerConfig_Config() {} -func (*TransformerConfig_CreditCardConfig) isTransformerConfig_Config() {} +func (*TransformerConfig_CardNumberConfig) isTransformerConfig_Config() {} func (*TransformerConfig_Sha256HashConfig) isTransformerConfig_Config() {} @@ -2242,7 +2242,7 @@ func (*FullAddress) Descriptor() ([]byte, []int) { return file_mgmt_v1alpha1_transformer_proto_rawDescGZIP(), []int{37} } -type CreditCard struct { +type CardNumber struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -2250,8 +2250,8 @@ type CreditCard struct { ValidLuhn bool `protobuf:"varint,1,opt,name=valid_luhn,json=validLuhn,proto3" json:"valid_luhn,omitempty"` } -func (x *CreditCard) Reset() { - *x = CreditCard{} +func (x *CardNumber) Reset() { + *x = CardNumber{} if protoimpl.UnsafeEnabled { mi := &file_mgmt_v1alpha1_transformer_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2259,13 +2259,13 @@ func (x *CreditCard) Reset() { } } -func (x *CreditCard) String() string { +func (x *CardNumber) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreditCard) ProtoMessage() {} +func (*CardNumber) ProtoMessage() {} -func (x *CreditCard) ProtoReflect() protoreflect.Message { +func (x *CardNumber) ProtoReflect() protoreflect.Message { mi := &file_mgmt_v1alpha1_transformer_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2277,12 +2277,12 @@ func (x *CreditCard) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreditCard.ProtoReflect.Descriptor instead. -func (*CreditCard) Descriptor() ([]byte, []int) { +// Deprecated: Use CardNumber.ProtoReflect.Descriptor instead. +func (*CardNumber) Descriptor() ([]byte, []int) { return file_mgmt_v1alpha1_transformer_proto_rawDescGZIP(), []int{38} } -func (x *CreditCard) GetValidLuhn() bool { +func (x *CardNumber) GetValidLuhn() bool { if x != nil { return x.ValidLuhn } @@ -2593,11 +2593,11 @@ var file_mgmt_v1alpha1_transformer_proto_rawDesc = []byte{ 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x46, 0x75, 0x6c, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x48, 0x00, 0x52, 0x11, 0x66, 0x75, 0x6c, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x49, 0x0a, - 0x12, 0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x5f, 0x63, 0x61, 0x72, 0x64, 0x5f, 0x63, 0x6f, 0x6e, + 0x12, 0x63, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x69, 0x74, - 0x43, 0x61, 0x72, 0x64, 0x48, 0x00, 0x52, 0x10, 0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x43, 0x61, - 0x72, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x48, 0x0a, 0x11, 0x73, 0x68, 0x61, 0x32, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x61, 0x72, 0x64, 0x4e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x48, 0x00, 0x52, 0x10, 0x63, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x48, 0x0a, 0x11, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x48, 0x61, 0x73, 0x68, 0x48, 0x00, @@ -2670,7 +2670,7 @@ var file_mgmt_v1alpha1_transformer_proto_rawDesc = []byte{ 0x72, 0x65, 0x73, 0x73, 0x22, 0x09, 0x0a, 0x07, 0x5a, 0x69, 0x70, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x43, 0x69, 0x74, 0x79, 0x22, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x0d, 0x0a, 0x0b, 0x46, 0x75, 0x6c, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, - 0x2b, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x69, 0x74, 0x43, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x0a, + 0x2b, 0x0a, 0x0a, 0x43, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x6c, 0x75, 0x68, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x4c, 0x75, 0x68, 0x6e, 0x22, 0x0c, 0x0a, 0x0a, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x48, 0x61, 0x73, 0x68, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x6f, @@ -2799,7 +2799,7 @@ var file_mgmt_v1alpha1_transformer_proto_goTypes = []interface{}{ (*City)(nil), // 35: mgmt.v1alpha1.City (*State)(nil), // 36: mgmt.v1alpha1.State (*FullAddress)(nil), // 37: mgmt.v1alpha1.FullAddress - (*CreditCard)(nil), // 38: mgmt.v1alpha1.CreditCard + (*CardNumber)(nil), // 38: mgmt.v1alpha1.CardNumber (*SHA256Hash)(nil), // 39: mgmt.v1alpha1.SHA256Hash (*SocialSecurityNumber)(nil), // 40: mgmt.v1alpha1.SocialSecurityNumber (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp @@ -2837,7 +2837,7 @@ var file_mgmt_v1alpha1_transformer_proto_depIdxs = []int32{ 34, // 29: mgmt.v1alpha1.TransformerConfig.zipcode_config:type_name -> mgmt.v1alpha1.Zipcode 36, // 30: mgmt.v1alpha1.TransformerConfig.state_config:type_name -> mgmt.v1alpha1.State 37, // 31: mgmt.v1alpha1.TransformerConfig.full_address_config:type_name -> mgmt.v1alpha1.FullAddress - 38, // 32: mgmt.v1alpha1.TransformerConfig.credit_card_config:type_name -> mgmt.v1alpha1.CreditCard + 38, // 32: mgmt.v1alpha1.TransformerConfig.card_number_config:type_name -> mgmt.v1alpha1.CardNumber 39, // 33: mgmt.v1alpha1.TransformerConfig.sha256hash_config:type_name -> mgmt.v1alpha1.SHA256Hash 40, // 34: mgmt.v1alpha1.TransformerConfig.ssn_config:type_name -> mgmt.v1alpha1.SocialSecurityNumber 0, // 35: mgmt.v1alpha1.TransformersService.GetSystemTransformers:input_type -> mgmt.v1alpha1.GetSystemTransformersRequest @@ -3324,7 +3324,7 @@ func file_mgmt_v1alpha1_transformer_proto_init() { } } file_mgmt_v1alpha1_transformer_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreditCard); i { + switch v := v.(*CardNumber); i { case 0: return &v.state case 1: @@ -3382,7 +3382,7 @@ func file_mgmt_v1alpha1_transformer_proto_init() { (*TransformerConfig_ZipcodeConfig)(nil), (*TransformerConfig_StateConfig)(nil), (*TransformerConfig_FullAddressConfig)(nil), - (*TransformerConfig_CreditCardConfig)(nil), + (*TransformerConfig_CardNumberConfig)(nil), (*TransformerConfig_Sha256HashConfig)(nil), (*TransformerConfig_SsnConfig)(nil), } diff --git a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.validate.go b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.validate.go index b367cc7cd9..5ebb1ec6f7 100644 --- a/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.validate.go +++ b/backend/gen/go/protos/mgmt/v1alpha1/transformer.pb.validate.go @@ -2939,7 +2939,7 @@ func (m *TransformerConfig) validate(all bool) error { } } - case *TransformerConfig_CreditCardConfig: + case *TransformerConfig_CardNumberConfig: if v == nil { err := TransformerConfigValidationError{ field: "Config", @@ -2952,11 +2952,11 @@ func (m *TransformerConfig) validate(all bool) error { } if all { - switch v := interface{}(m.GetCreditCardConfig()).(type) { + switch v := interface{}(m.GetCardNumberConfig()).(type) { case interface{ ValidateAll() error }: if err := v.ValidateAll(); err != nil { errors = append(errors, TransformerConfigValidationError{ - field: "CreditCardConfig", + field: "CardNumberConfig", reason: "embedded message failed validation", cause: err, }) @@ -2964,16 +2964,16 @@ func (m *TransformerConfig) validate(all bool) error { case interface{ Validate() error }: if err := v.Validate(); err != nil { errors = append(errors, TransformerConfigValidationError{ - field: "CreditCardConfig", + field: "CardNumberConfig", reason: "embedded message failed validation", cause: err, }) } } - } else if v, ok := interface{}(m.GetCreditCardConfig()).(interface{ Validate() error }); ok { + } else if v, ok := interface{}(m.GetCardNumberConfig()).(interface{ Validate() error }); ok { if err := v.Validate(); err != nil { return TransformerConfigValidationError{ - field: "CreditCardConfig", + field: "CardNumberConfig", reason: "embedded message failed validation", cause: err, } @@ -5258,22 +5258,22 @@ var _ interface { ErrorName() string } = FullAddressValidationError{} -// Validate checks the field values on CreditCard with the rules defined in the +// Validate checks the field values on CardNumber with the rules defined in the // proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. -func (m *CreditCard) Validate() error { +func (m *CardNumber) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on CreditCard with the rules defined in +// ValidateAll checks the field values on CardNumber with the rules defined in // the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in CreditCardMultiError, or +// result is a list of violation errors wrapped in CardNumberMultiError, or // nil if none found. -func (m *CreditCard) ValidateAll() error { +func (m *CardNumber) ValidateAll() error { return m.validate(true) } -func (m *CreditCard) validate(all bool) error { +func (m *CardNumber) validate(all bool) error { if m == nil { return nil } @@ -5283,18 +5283,18 @@ func (m *CreditCard) validate(all bool) error { // no validation rules for ValidLuhn if len(errors) > 0 { - return CreditCardMultiError(errors) + return CardNumberMultiError(errors) } return nil } -// CreditCardMultiError is an error wrapping multiple validation errors -// returned by CreditCard.ValidateAll() if the designated constraints aren't met. -type CreditCardMultiError []error +// CardNumberMultiError is an error wrapping multiple validation errors +// returned by CardNumber.ValidateAll() if the designated constraints aren't met. +type CardNumberMultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m CreditCardMultiError) Error() string { +func (m CardNumberMultiError) Error() string { var msgs []string for _, err := range m { msgs = append(msgs, err.Error()) @@ -5303,11 +5303,11 @@ func (m CreditCardMultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m CreditCardMultiError) AllErrors() []error { return m } +func (m CardNumberMultiError) AllErrors() []error { return m } -// CreditCardValidationError is the validation error returned by -// CreditCard.Validate if the designated constraints aren't met. -type CreditCardValidationError struct { +// CardNumberValidationError is the validation error returned by +// CardNumber.Validate if the designated constraints aren't met. +type CardNumberValidationError struct { field string reason string cause error @@ -5315,22 +5315,22 @@ type CreditCardValidationError struct { } // Field function returns field value. -func (e CreditCardValidationError) Field() string { return e.field } +func (e CardNumberValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e CreditCardValidationError) Reason() string { return e.reason } +func (e CardNumberValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e CreditCardValidationError) Cause() error { return e.cause } +func (e CardNumberValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e CreditCardValidationError) Key() bool { return e.key } +func (e CardNumberValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e CreditCardValidationError) ErrorName() string { return "CreditCardValidationError" } +func (e CardNumberValidationError) ErrorName() string { return "CardNumberValidationError" } // Error satisfies the builtin error interface -func (e CreditCardValidationError) Error() string { +func (e CardNumberValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -5342,14 +5342,14 @@ func (e CreditCardValidationError) Error() string { } return fmt.Sprintf( - "invalid %sCreditCard.%s: %s%s", + "invalid %sCardNumber.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = CreditCardValidationError{} +var _ error = CardNumberValidationError{} var _ interface { Field() string @@ -5357,7 +5357,7 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = CreditCardValidationError{} +} = CardNumberValidationError{} // Validate checks the field values on SHA256Hash with the rules defined in the // proto definition for this message. If any rules are violated, the first diff --git a/backend/protos/mgmt/v1alpha1/transformer.proto b/backend/protos/mgmt/v1alpha1/transformer.proto index 9018c3d299..b100bae281 100644 --- a/backend/protos/mgmt/v1alpha1/transformer.proto +++ b/backend/protos/mgmt/v1alpha1/transformer.proto @@ -108,7 +108,7 @@ message TransformerConfig { Zipcode zipcode_config = 19; State state_config = 20; FullAddress full_address_config = 21; - CreditCard credit_card_config = 22; + CardNumber card_number_config = 22; SHA256Hash sha256hash_config = 23; SocialSecurityNumber ssn_config = 24; } @@ -183,7 +183,7 @@ message State {} message FullAddress {} -message CreditCard { +message CardNumber { bool valid_luhn = 1; } diff --git a/backend/services/mgmt/v1alpha1/transformers-service/transformers.go b/backend/services/mgmt/v1alpha1/transformers-service/transformers.go index 79c201bb0c..5b76a8deef 100644 --- a/backend/services/mgmt/v1alpha1/transformers-service/transformers.go +++ b/backend/services/mgmt/v1alpha1/transformers-service/transformers.go @@ -38,7 +38,7 @@ const ( Zipcode Transformation = "zipcode" State Transformation = "state" FullAddress Transformation = "full_address" - CreditCard Transformation = "credit_card" + CardNumber Transformation = "card_number" SHA256 Transformation = "sha256_hash" SocialSecurityNumber Transformation = "social_security_number" ) @@ -190,9 +190,9 @@ func (s *Service) GetSystemTransformers( FullAddressConfig: &mgmtv1alpha1.FullAddress{}, }, }}, - {Value: string(CreditCard), Config: &mgmtv1alpha1.TransformerConfig{ - Config: &mgmtv1alpha1.TransformerConfig_CreditCardConfig{ - CreditCardConfig: &mgmtv1alpha1.CreditCard{ + {Value: string(CardNumber), Config: &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_CardNumberConfig{ + CardNumberConfig: &mgmtv1alpha1.CardNumber{ ValidLuhn: true, }, }, diff --git a/backend/sql/postgresql/models/transformers.go b/backend/sql/postgresql/models/transformers.go index 56518a26b9..58aac1350d 100644 --- a/backend/sql/postgresql/models/transformers.go +++ b/backend/sql/postgresql/models/transformers.go @@ -31,7 +31,7 @@ type TransformerConfigs struct { Zipcode *ZipcodeConfig `json:"zipcode,omitempty"` State *StateConfig `json:"state,omitempty"` FullAddress *FullAddressConfig `json:"fullAddress,omitempty"` - CreditCard *CreditCardConfig `json:"creditcard,omitempty"` + CardNumber *CardNumberConfig `json:"cardnumber,omitempty"` SHA256Hash *SHA256HashConfig `json:"sha256Hash,omitempty"` SocialSecurityNumber *SocialSecurityNumberConfig `json:"social_security_number,omitempty"` } @@ -105,7 +105,7 @@ type StateConfig struct{} type FullAddressConfig struct{} -type CreditCardConfig struct { +type CardNumberConfig struct { ValidLuhn bool `json:"validLuhn"` } @@ -211,14 +211,17 @@ func (t *TransformerConfigs) FromTransformerConfigDto(tr *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_FullAddressConfig: t.FullAddress = &FullAddressConfig{} - case *mgmtv1alpha1.TransformerConfig_CreditCardConfig: - t.CreditCard = &CreditCardConfig{ - ValidLuhn: tr.GetCreditCardConfig().ValidLuhn, + + case *mgmtv1alpha1.TransformerConfig_CardNumberConfig: + t.CardNumber = &CardNumberConfig{ + ValidLuhn: tr.GetCardNumberConfig().ValidLuhn, } case *mgmtv1alpha1.TransformerConfig_Sha256HashConfig: t.SHA256Hash = &SHA256HashConfig{} + case *mgmtv1alpha1.TransformerConfig_SsnConfig: t.SocialSecurityNumber = &SocialSecurityNumberConfig{} + default: t = &TransformerConfigs{} } @@ -396,11 +399,11 @@ func (t *TransformerConfigs) ToTransformerConfigDto(tr *TransformerConfigs) *mgm FullAddressConfig: &mgmtv1alpha1.FullAddress{}, }, } - case tr.CreditCard != nil: + case tr.CardNumber != nil: return &mgmtv1alpha1.TransformerConfig{ - Config: &mgmtv1alpha1.TransformerConfig_CreditCardConfig{ - CreditCardConfig: &mgmtv1alpha1.CreditCard{ - ValidLuhn: tr.CreditCard.ValidLuhn, + Config: &mgmtv1alpha1.TransformerConfig_CardNumberConfig{ + CardNumberConfig: &mgmtv1alpha1.CardNumber{ + ValidLuhn: tr.CardNumber.ValidLuhn, }, }, } diff --git a/docs/docs/transformers/system/card-number.mdx b/docs/docs/transformers/system/card-number.mdx index 97b83ca577..43d6c9dd6e 100644 --- a/docs/docs/transformers/system/card-number.mdx +++ b/docs/docs/transformers/system/card-number.mdx @@ -12,12 +12,12 @@ import { DocsTable } from '@site/src/CustomComponents/DocsTable.tsx'; ## Definition -The credit card transformer can anonymize an existing email address or completely generate a new one. A card number can be quite complex but we make it easy to generate valid luhn-check card numbers. +The card number transformer can anonymize an existing email address or completely generate a new one. A card number can be quite complex but we make it easy to generate valid luhn-check card numbers. By default, the card number transformer generates a random 16 digit card number that is _not_ luhn valid. If you want luhn validation, please set the luhn-check config to `true`. diff --git a/frontend/app/new/transformer/CustomTransformerForms/CustomCreditCardTransformerForm.tsx b/frontend/app/new/transformer/CustomTransformerForms/CustomCardNumberTransformerForm.tsx similarity index 89% rename from frontend/app/new/transformer/CustomTransformerForms/CustomCreditCardTransformerForm.tsx rename to frontend/app/new/transformer/CustomTransformerForms/CustomCardNumberTransformerForm.tsx index bdefaf7b44..372d8839bb 100644 --- a/frontend/app/new/transformer/CustomTransformerForms/CustomCreditCardTransformerForm.tsx +++ b/frontend/app/new/transformer/CustomTransformerForms/CustomCardNumberTransformerForm.tsx @@ -15,7 +15,7 @@ interface Props { isDisabled?: boolean; } -export default function CustomCreditCardTransformerForm( +export default function CustomCardNumberTransformerForm( props: Props ): ReactElement { const fc = useFormContext(); @@ -32,7 +32,7 @@ export default function CustomCreditCardTransformerForm(
Valid Luhn - Generate a 16 digit credit card number that passes a luhn check. + Generate a 16 digit card number that passes a luhn check.
diff --git a/frontend/app/new/transformer/CustomTransformerForms/HandleCustomTransformersForm.tsx b/frontend/app/new/transformer/CustomTransformerForms/HandleCustomTransformersForm.tsx index 914762fbbf..7e4e74a5ae 100644 --- a/frontend/app/new/transformer/CustomTransformerForms/HandleCustomTransformersForm.tsx +++ b/frontend/app/new/transformer/CustomTransformerForms/HandleCustomTransformersForm.tsx @@ -1,5 +1,5 @@ import { ReactElement } from 'react'; -import CustomCreditCardTransformerForm from './CustomCreditCardTransformerForm'; +import CustomCardNumberTransformerForm from './CustomCardNumberTransformerForm'; import CustomEmailTransformerForm from './CustomEmailTransformerForm'; import CustomFirstNameTransformerForm from './CustomFirstnameTransformerForm'; import CustomFullNameTransformerForm from './CustomFullnameTransformerForm'; @@ -40,8 +40,8 @@ export function handleCustomTransformerForm( return ; case 'gender': return ; - case 'credit_card': - return ; + case 'card_number': + return ; default:
No transformer component found
; } diff --git a/frontend/app/new/transformer/schema.ts b/frontend/app/new/transformer/schema.ts index 821b25ac62..c57ab52a40 100644 --- a/frontend/app/new/transformer/schema.ts +++ b/frontend/app/new/transformer/schema.ts @@ -51,7 +51,7 @@ const randomFloatConfig = Yup.object().shape({ digitsBeforeDecimal: Yup.number().notRequired(), }); -const creditcardConfig = Yup.object().shape({ +const cardNumberConfig = Yup.object().shape({ validLuhn: Yup.boolean().notRequired(), }); @@ -103,10 +103,12 @@ export const transformerConfig = Yup.object().shape({ return Yup.object().shape({}); case 'streetAddressConfig': return Yup.object().shape({}); - case 'creditcardConfig': - return creditcardConfig; + case 'cardNumberConfig': + return cardNumberConfig; case 'sha256hashConfig': return Yup.object().shape({}); + case 'ssnConfig': + return Yup.object().shape({}); default: return Yup.object().shape({}); } diff --git a/frontend/app/transformers/EditTransformerOptions.tsx b/frontend/app/transformers/EditTransformerOptions.tsx index f3f6963621..b8541993b4 100644 --- a/frontend/app/transformers/EditTransformerOptions.tsx +++ b/frontend/app/transformers/EditTransformerOptions.tsx @@ -21,7 +21,7 @@ import { Pencil1Icon, } from '@radix-ui/react-icons'; import { ReactElement, useEffect, useRef, useState } from 'react'; -import CreditCardTransformerForm from './Sheetforms/CreditCardTransformerForm'; +import CardNumberTransformerForm from './Sheetforms/CardNumberTransformerForm'; import EmailTransformerForm from './Sheetforms/EmailTransformerForm'; import FirstNameTransformerForm from './Sheetforms/FirstnameTransformerForm'; import FullNameTransformerForm from './Sheetforms/FullnameTransformerForm'; @@ -211,9 +211,9 @@ function handleTransformerForm( transformer={transformer} /> ); - case 'credit_card': + case 'card_number': return ( - void; } -export default function CreditCardTransformerForm(props: Props): ReactElement { +export default function CardNumberTransformerForm(props: Props): ReactElement { const { index, setIsSheetOpen, transformer } = props; const fc = useFormContext(); - const config = transformer?.config?.config.value as CreditCard; + const config = transformer?.config?.config.value as CardNumber; const [vl, setVl] = useState( config?.validLuhn ? config?.validLuhn : false @@ -53,7 +53,7 @@ export default function CreditCardTransformerForm(props: Props): ReactElement {
Valid Luhn - Generate a 16 digit credit card number that passes a luhn check. + Generate a 16 digit card number that passes a luhn check.
diff --git a/frontend/neosync-api-client/mgmt/v1alpha1/transformer_pb.ts b/frontend/neosync-api-client/mgmt/v1alpha1/transformer_pb.ts index 1def4c484b..7db103569e 100644 --- a/frontend/neosync-api-client/mgmt/v1alpha1/transformer_pb.ts +++ b/frontend/neosync-api-client/mgmt/v1alpha1/transformer_pb.ts @@ -829,10 +829,10 @@ export class TransformerConfig extends Message { case: "fullAddressConfig"; } | { /** - * @generated from field: mgmt.v1alpha1.CreditCard credit_card_config = 22; + * @generated from field: mgmt.v1alpha1.CardNumber card_number_config = 22; */ - value: CreditCard; - case: "creditCardConfig"; + value: CardNumber; + case: "cardNumberConfig"; } | { /** * @generated from field: mgmt.v1alpha1.SHA256Hash sha256hash_config = 23; @@ -876,7 +876,7 @@ export class TransformerConfig extends Message { { no: 19, name: "zipcode_config", kind: "message", T: Zipcode, oneof: "config" }, { no: 20, name: "state_config", kind: "message", T: State, oneof: "config" }, { no: 21, name: "full_address_config", kind: "message", T: FullAddress, oneof: "config" }, - { no: 22, name: "credit_card_config", kind: "message", T: CreditCard, oneof: "config" }, + { no: 22, name: "card_number_config", kind: "message", T: CardNumber, oneof: "config" }, { no: 23, name: "sha256hash_config", kind: "message", T: SHA256Hash, oneof: "config" }, { no: 24, name: "ssn_config", kind: "message", T: SocialSecurityNumber, oneof: "config" }, ]); @@ -1658,39 +1658,39 @@ export class FullAddress extends Message { } /** - * @generated from message mgmt.v1alpha1.CreditCard + * @generated from message mgmt.v1alpha1.CardNumber */ -export class CreditCard extends Message { +export class CardNumber extends Message { /** * @generated from field: bool valid_luhn = 1; */ validLuhn = false; - constructor(data?: PartialMessage) { + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); } static readonly runtime: typeof proto3 = proto3; - static readonly typeName = "mgmt.v1alpha1.CreditCard"; + static readonly typeName = "mgmt.v1alpha1.CardNumber"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "valid_luhn", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, ]); - static fromBinary(bytes: Uint8Array, options?: Partial): CreditCard { - return new CreditCard().fromBinary(bytes, options); + static fromBinary(bytes: Uint8Array, options?: Partial): CardNumber { + return new CardNumber().fromBinary(bytes, options); } - static fromJson(jsonValue: JsonValue, options?: Partial): CreditCard { - return new CreditCard().fromJson(jsonValue, options); + static fromJson(jsonValue: JsonValue, options?: Partial): CardNumber { + return new CardNumber().fromJson(jsonValue, options); } - static fromJsonString(jsonString: string, options?: Partial): CreditCard { - return new CreditCard().fromJsonString(jsonString, options); + static fromJsonString(jsonString: string, options?: Partial): CardNumber { + return new CardNumber().fromJsonString(jsonString, options); } - static equals(a: CreditCard | PlainMessage | undefined, b: CreditCard | PlainMessage | undefined): boolean { - return proto3.util.equals(CreditCard, a, b); + static equals(a: CardNumber | PlainMessage | undefined, b: CardNumber | PlainMessage | undefined): boolean { + return proto3.util.equals(CardNumber, a, b); } } diff --git a/frontend/yup-validations/transformers.ts b/frontend/yup-validations/transformers.ts index ad5bc303c0..300f7349ba 100644 --- a/frontend/yup-validations/transformers.ts +++ b/frontend/yup-validations/transformers.ts @@ -1,6 +1,6 @@ import { + CardNumber, City, - CreditCard, CustomTransformer, EmailConfig, FirstName, @@ -134,12 +134,12 @@ interface GenderTransformerConfigs { abbreviate: boolean; } -interface CreditCardTransformer { +interface CardNumberTransformer { case?: string | undefined; - value: CreditCardTransformerConfigs; + value: CardNumberTransformerConfigs; } -interface CreditCardTransformerConfigs { +interface CardNumberTransformerConfigs { validLuhn: boolean; } @@ -428,14 +428,14 @@ export function ToTransformerConfigOptions( }), }); } - case 'creditcard': { - const g = t.config.config as CreditCardTransformer; + case 'card_number': { + const g = t.config.config as CardNumberTransformer; return new Transformer({ value: val.source, config: new TransformerConfig({ config: { - case: 'creditCardConfig', - value: new CreditCard({ + case: 'cardNumberConfig', + value: new CardNumber({ validLuhn: g.value.validLuhn, }), }, diff --git a/worker/internal/benthos/transformers/credit_card.go b/worker/internal/benthos/transformers/card_number.go similarity index 81% rename from worker/internal/benthos/transformers/credit_card.go rename to worker/internal/benthos/transformers/card_number.go index 8f9fc11ece..a89134d15a 100644 --- a/worker/internal/benthos/transformers/credit_card.go +++ b/worker/internal/benthos/transformers/card_number.go @@ -19,7 +19,7 @@ func init() { Param(bloblang.NewBoolParam("luhn_check")) // register the plugin - err := bloblang.RegisterFunctionV2("creditcardtransformer", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { + err := bloblang.RegisterFunctionV2("cardnumbertransformer", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { luhn, err := args.GetBool("luhn_check") if err != nil { @@ -27,7 +27,7 @@ func init() { } return func() (any, error) { - res, err := GenerateCreditCard(luhn) + res, err := GenerateCardNumber(luhn) return res, err }, nil }) @@ -39,15 +39,15 @@ func init() { } // main transformer logic goes here -func GenerateCreditCard(luhn bool) (int64, error) { +func GenerateCardNumber(luhn bool) (int64, error) { var returnValue int64 if luhn { - val, err := GenerateValidVLuhnCheckCreditCard() + val, err := GenerateValidVLuhnCheckCardNumber() if err != nil { - return 0, fmt.Errorf("unable to generate a luhn valid credit card number") + return 0, fmt.Errorf("unable to generate a luhn valid card number") } returnValue = val @@ -57,7 +57,7 @@ func GenerateCreditCard(luhn bool) (int64, error) { val, err := transformer_utils.GenerateRandomInt(defualtCCLength) if err != nil { - return 0, fmt.Errorf("unable to generate a random credit card number") + return 0, fmt.Errorf("unable to generate a random card number") } returnValue = val @@ -67,8 +67,8 @@ func GenerateCreditCard(luhn bool) (int64, error) { return returnValue, nil } -// generates a credit card number that passes luhn validation -func GenerateValidVLuhnCheckCreditCard() (int64, error) { +// generates a card number that passes luhn validation +func GenerateValidVLuhnCheckCardNumber() (int64, error) { // To find the checksum digit on cardNo := make([]int, 0) diff --git a/worker/internal/benthos/transformers/credit_card_test.go b/worker/internal/benthos/transformers/card_number_test.go similarity index 52% rename from worker/internal/benthos/transformers/credit_card_test.go rename to worker/internal/benthos/transformers/card_number_test.go index fd0f717607..40e0e89f1d 100644 --- a/worker/internal/benthos/transformers/credit_card_test.go +++ b/worker/internal/benthos/transformers/card_number_test.go @@ -8,34 +8,34 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGenerateValidLuhnCreditCard(t *testing.T) { +func TestGenerateValidLuhnCardNumber(t *testing.T) { - val, err := GenerateValidVLuhnCheckCreditCard() + val, err := GenerateValidVLuhnCheckCardNumber() assert.NoError(t, err) - assert.Len(t, strconv.FormatInt(val, 10), 16, "The output credit card should be 16 characteres long") - assert.Equal(t, isValidLuhn(val), true) + assert.Len(t, strconv.FormatInt(val, 10), 16, "The output card should be 16 characters long") + assert.Equal(t, true, isValidLuhn(val)) } -func TestGenerateRandomLuhnCreditCard(t *testing.T) { +func TestGenerateRandomLuhnCardNumber(t *testing.T) { - val, err := GenerateCreditCard(false) + val, err := GenerateCardNumber(false) assert.NoError(t, err) - assert.Len(t, strconv.FormatInt(val, 10), 16, "The output credit card should be 16 characteres long") - assert.Equal(t, isValidLuhn(val), false) + assert.Len(t, strconv.FormatInt(val, 10), 16, "The output card should be 16 characters long") + assert.Equal(t, false, isValidLuhn(val)) } -func TestGenerateCreditCardTransformer(t *testing.T) { - mapping := `root = creditcardtransformer(true)` +func TestGenerateCardNumberTransformer(t *testing.T) { + mapping := `root = cardnumbertransformer(true)` ex, err := bloblang.Parse(mapping) assert.NoError(t, err, "failed to parse the random int transformer") res, err := ex.Query(nil) assert.NoError(t, err) - assert.Len(t, strconv.FormatInt(res.(int64), 10), 16, "The output credit card should be 16 characteres long") - assert.Equal(t, isValidLuhn(res.(int64)), true) + assert.Len(t, strconv.FormatInt(res.(int64), 10), 16, "The output card should be 16 characters long") + assert.Equal(t, true, isValidLuhn(res.(int64))) } func isValidLuhn(cc int64) bool { diff --git a/worker/pkg/workflows/datasync/activities/activities.go b/worker/pkg/workflows/datasync/activities/activities.go index 43b3944ac6..22bbe2588f 100644 --- a/worker/pkg/workflows/datasync/activities/activities.go +++ b/worker/pkg/workflows/datasync/activities/activities.go @@ -666,9 +666,9 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping) (string, error) { return "statetransformer()", nil case "full_address": return "fulladdresstransformer()", nil - case "credit_card": - luhn := col.Transformer.Config.GetCreditCardConfig().ValidLuhn - return fmt.Sprintf(`creditcardtransformer(%t)`, luhn), nil + case "card_number": + luhn := col.Transformer.Config.GetCardNumberConfig().ValidLuhn + return fmt.Sprintf(`cardnumbertransformer(%t)`, luhn), nil case "sha256": return fmt.Sprintf(`this.%s.bytes().hash("sha256").encode("hex")`, col.Column), nil case "social_security_number": From 1a81ceff23f07c1811d4a7dc6e8192ab050c598f Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 14 Nov 2023 15:02:25 -0800 Subject: [PATCH 2/5] updates to test --- worker/internal/benthos/transformers/card_number.go | 10 +++++++--- .../internal/benthos/transformers/card_number_test.go | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/worker/internal/benthos/transformers/card_number.go b/worker/internal/benthos/transformers/card_number.go index a89134d15a..808306fd30 100644 --- a/worker/internal/benthos/transformers/card_number.go +++ b/worker/internal/benthos/transformers/card_number.go @@ -2,7 +2,6 @@ package neosync_transformers import ( "fmt" - "math/rand" "strconv" "github.com/benthosdev/benthos/v4/public/bloblang" @@ -83,8 +82,13 @@ func GenerateValidVLuhnCheckCardNumber() (int64, error) { } // Acc no (9 digits) - seventh15 := rand.Perm(9)[:9] - for _, i := range seventh15 { + nineDigits, _ := transformer_utils.GenerateRandomInt(int64(9)) + strNine := strconv.FormatInt(nineDigits, 10) + var nineInts []int + for _, k := range strNine { + nineInts = append(nineInts, int(k)) + } + for _, i := range nineInts { cardNo = append(cardNo, i) cardNum = append(cardNum, i) } diff --git a/worker/internal/benthos/transformers/card_number_test.go b/worker/internal/benthos/transformers/card_number_test.go index 40e0e89f1d..60d8622060 100644 --- a/worker/internal/benthos/transformers/card_number_test.go +++ b/worker/internal/benthos/transformers/card_number_test.go @@ -1,6 +1,7 @@ package neosync_transformers import ( + "fmt" "strconv" "testing" @@ -17,13 +18,15 @@ func TestGenerateValidLuhnCardNumber(t *testing.T) { assert.Equal(t, true, isValidLuhn(val)) } -func TestGenerateRandomLuhnCardNumber(t *testing.T) { +func TestGenerateCardNumber(t *testing.T) { val, err := GenerateCardNumber(false) + fmt.Println("val", val) + fmt.Println("luhn check", isValidLuhn((val))) + assert.NoError(t, err) assert.Len(t, strconv.FormatInt(val, 10), 16, "The output card should be 16 characters long") - assert.Equal(t, false, isValidLuhn(val)) } func TestGenerateCardNumberTransformer(t *testing.T) { From 7365ba30fa1042ef9a6688b7fa4706a718d87045 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 14 Nov 2023 15:05:01 -0800 Subject: [PATCH 3/5] clean up --- worker/internal/benthos/transformers/card_number_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/worker/internal/benthos/transformers/card_number_test.go b/worker/internal/benthos/transformers/card_number_test.go index 60d8622060..932383e115 100644 --- a/worker/internal/benthos/transformers/card_number_test.go +++ b/worker/internal/benthos/transformers/card_number_test.go @@ -1,7 +1,6 @@ package neosync_transformers import ( - "fmt" "strconv" "testing" @@ -22,9 +21,6 @@ func TestGenerateCardNumber(t *testing.T) { val, err := GenerateCardNumber(false) - fmt.Println("val", val) - fmt.Println("luhn check", isValidLuhn((val))) - assert.NoError(t, err) assert.Len(t, strconv.FormatInt(val, 10), 16, "The output card should be 16 characters long") } From 4c579ecaa86282b13517c30e7e5cca38782a22f6 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 14 Nov 2023 15:38:55 -0800 Subject: [PATCH 4/5] updated luhn check --- .../benthos/transformers/card_number.go | 22 +++++++++++-------- .../benthos/transformers/card_number_test.go | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/worker/internal/benthos/transformers/card_number.go b/worker/internal/benthos/transformers/card_number.go index 808306fd30..4bb0ddac44 100644 --- a/worker/internal/benthos/transformers/card_number.go +++ b/worker/internal/benthos/transformers/card_number.go @@ -43,7 +43,7 @@ func GenerateCardNumber(luhn bool) (int64, error) { if luhn { - val, err := GenerateValidVLuhnCheckCardNumber() + val, err := GenerateValidLuhnCheckCardNumber() if err != nil { return 0, fmt.Errorf("unable to generate a luhn valid card number") @@ -67,7 +67,7 @@ func GenerateCardNumber(luhn bool) (int64, error) { } // generates a card number that passes luhn validation -func GenerateValidVLuhnCheckCardNumber() (int64, error) { +func GenerateValidLuhnCheckCardNumber() (int64, error) { // To find the checksum digit on cardNo := make([]int, 0) @@ -82,15 +82,19 @@ func GenerateValidVLuhnCheckCardNumber() (int64, error) { } // Acc no (9 digits) - nineDigits, _ := transformer_utils.GenerateRandomInt(int64(9)) + nineDigits, err := transformer_utils.GenerateRandomInt(int64(9)) + if err != nil { + return 0, err + } strNine := strconv.FormatInt(nineDigits, 10) - var nineInts []int for _, k := range strNine { - nineInts = append(nineInts, int(k)) - } - for _, i := range nineInts { - cardNo = append(cardNo, i) - cardNum = append(cardNum, i) + digit, err := strconv.Atoi(string(k)) + if err != nil { + fmt.Println("Error converting to int:", err) + continue + } + cardNo = append(cardNo, digit) + cardNum = append(cardNum, digit) } // odd position digits diff --git a/worker/internal/benthos/transformers/card_number_test.go b/worker/internal/benthos/transformers/card_number_test.go index 932383e115..cb406c8897 100644 --- a/worker/internal/benthos/transformers/card_number_test.go +++ b/worker/internal/benthos/transformers/card_number_test.go @@ -10,7 +10,7 @@ import ( func TestGenerateValidLuhnCardNumber(t *testing.T) { - val, err := GenerateValidVLuhnCheckCardNumber() + val, err := GenerateValidLuhnCheckCardNumber() assert.NoError(t, err) assert.Len(t, strconv.FormatInt(val, 10), 16, "The output card should be 16 characters long") From e09183ae4cb36da5f431e7e6628b261e4ae2d951 Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Tue, 14 Nov 2023 15:41:07 -0800 Subject: [PATCH 5/5] clean up --- worker/internal/benthos/transformers/card_number.go | 1 - 1 file changed, 1 deletion(-) diff --git a/worker/internal/benthos/transformers/card_number.go b/worker/internal/benthos/transformers/card_number.go index 4bb0ddac44..44ad0caf7f 100644 --- a/worker/internal/benthos/transformers/card_number.go +++ b/worker/internal/benthos/transformers/card_number.go @@ -90,7 +90,6 @@ func GenerateValidLuhnCheckCardNumber() (int64, error) { for _, k := range strNine { digit, err := strconv.Atoi(string(k)) if err != nil { - fmt.Println("Error converting to int:", err) continue } cardNo = append(cardNo, digit)