From a8f9c6407de201364536bd29fdf85433afc6473e Mon Sep 17 00:00:00 2001 From: Evis Drenova Date: Mon, 20 Nov 2023 22:24:35 -0800 Subject: [PATCH] refactored the sha256 transformer --- .../benthos/transformers/random_float.go | 3 + .../benthos/transformers/random_float_test.go | 13 ++++ .../benthos/transformers/random_int.go | 12 +++- .../benthos/transformers/random_int_test.go | 12 ++++ .../benthos/transformers/random_string.go | 10 +++ .../transformers/random_string_test.go | 12 ++++ .../benthos/transformers/sha256hash.go | 65 +++++++++++++++++++ .../benthos/transformers/sha256hash_test.go | 15 +++++ .../datasync/activities/activities.go | 8 ++- 9 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 worker/internal/benthos/transformers/sha256hash.go create mode 100644 worker/internal/benthos/transformers/sha256hash_test.go diff --git a/worker/internal/benthos/transformers/random_float.go b/worker/internal/benthos/transformers/random_float.go index 6619ff9277..850a2ab30d 100644 --- a/worker/internal/benthos/transformers/random_float.go +++ b/worker/internal/benthos/transformers/random_float.go @@ -85,6 +85,9 @@ func GenerateRandomFloat(value float64, preserveLength bool, digitsAfterDecimal, res, err := GenerateRandomFloatWithDefinedLength(digitsBeforeDecimal, digitsAfterDecimal) return res, err } + } else if digitsBeforeDecimal != 0 && digitsAfterDecimal != 0 { + res, err := GenerateRandomFloatWithDefinedLength(digitsBeforeDecimal, digitsAfterDecimal) + return res, err } else { res, err := GenerateRandomFloatWithRandomLength() return res, err diff --git a/worker/internal/benthos/transformers/random_float_test.go b/worker/internal/benthos/transformers/random_float_test.go index c6b25c299f..135ca39202 100644 --- a/worker/internal/benthos/transformers/random_float_test.go +++ b/worker/internal/benthos/transformers/random_float_test.go @@ -68,6 +68,19 @@ func TestRandomFloatTransformerWithNoValue(t *testing.T) { assert.Equal(t, 6, actual, "The length of the output float needs to match the digits before + the digits after") } +func TestRandomFloatTransformerWithNoValueAndLength(t *testing.T) { + mapping := `root = randomfloattransformer(digits_before_decimal: 2, digits_after_decimal: 2)` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the random float transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + actual := GetFloatLength(res.(float64)).DigitsAfterDecimalLength + GetFloatLength(res.(float64)).DigitsBeforeDecimalLength + assert.IsType(t, res, float64(1)) + assert.Equal(t, 4, actual, "The length of the output float needs to match the digits before + the digits after") +} + func Test_GetFloatLength(t *testing.T) { val := float64(3.14) res := GetFloatLength(val) diff --git a/worker/internal/benthos/transformers/random_int.go b/worker/internal/benthos/transformers/random_int.go index 5ee4496c50..824ed099ec 100644 --- a/worker/internal/benthos/transformers/random_int.go +++ b/worker/internal/benthos/transformers/random_int.go @@ -101,9 +101,9 @@ func GenerateRandomInt(value int64, preserveLength bool, intLength int64) (int64 returnValue = val } - } else { + } else if intLength != 0 { - val, err := transformer_utils.GenerateRandomInt(defaultIntLength) + val, err := transformer_utils.GenerateRandomInt(intLength) if err != nil { return 0, fmt.Errorf("unable to generate a random string with length") @@ -111,6 +111,14 @@ func GenerateRandomInt(value int64, preserveLength bool, intLength int64) (int64 returnValue = val + } else { + val, err := transformer_utils.GenerateRandomInt(defaultIntLength) + + if err != nil { + return 0, fmt.Errorf("unable to generate a random string with length") + } + + returnValue = val } return returnValue, nil diff --git a/worker/internal/benthos/transformers/random_int_test.go b/worker/internal/benthos/transformers/random_int_test.go index 60184e6e42..cc0b0726e4 100644 --- a/worker/internal/benthos/transformers/random_int_test.go +++ b/worker/internal/benthos/transformers/random_int_test.go @@ -79,3 +79,15 @@ func TestRandomIntTransformerWithNoValue(t *testing.T) { assert.Equal(t, int64(4), transformer_utils.GetIntLength(res.(int64))) // Generated int must be the same length as the input int" assert.IsType(t, res, int64(2)) } + +func TestRandomIntTransformerWithNoValueAndLength(t *testing.T) { + mapping := `root = randominttransformer(int_length: 4)` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the random int transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + assert.Equal(t, int64(4), transformer_utils.GetIntLength(res.(int64))) // Generated int must be the same length as the input int" + assert.IsType(t, res, int64(2)) +} diff --git a/worker/internal/benthos/transformers/random_string.go b/worker/internal/benthos/transformers/random_string.go index 8776ea3235..885b5a1a85 100644 --- a/worker/internal/benthos/transformers/random_string.go +++ b/worker/internal/benthos/transformers/random_string.go @@ -107,6 +107,16 @@ func GenerateRandomString(value string, preserveLength bool, strLength int64) (s returnValue = val } + } else if strLength != 0 { + + val, err := transformer_utils.GenerateRandomStringWithLength(strLength) + + if err != nil { + return "", fmt.Errorf("unable to generate a random string with length") + } + + returnValue = val + } else { val, err := transformer_utils.GenerateRandomStringWithLength(defaultStrLength) diff --git a/worker/internal/benthos/transformers/random_string_test.go b/worker/internal/benthos/transformers/random_string_test.go index b79b560bb8..755c1454bc 100644 --- a/worker/internal/benthos/transformers/random_string_test.go +++ b/worker/internal/benthos/transformers/random_string_test.go @@ -67,3 +67,15 @@ func TestRandomStringTransformerWithNoValue(t *testing.T) { assert.Equal(t, 10, len(res.(string)), "Generated string must be the same length as the input string") assert.IsType(t, res, "") } + +func TestRandomStringTransformerWithNoValueAndLength(t *testing.T) { + mapping := `root = randomstringtransformer(str_length:6)` + ex, err := bloblang.Parse(mapping) + assert.NoError(t, err, "failed to parse the random string transformer") + + res, err := ex.Query(nil) + assert.NoError(t, err) + + assert.Equal(t, 6, len(res.(string)), "Generated string must be the same length as the input string") + assert.IsType(t, res, "") +} diff --git a/worker/internal/benthos/transformers/sha256hash.go b/worker/internal/benthos/transformers/sha256hash.go new file mode 100644 index 0000000000..3ba7f88263 --- /dev/null +++ b/worker/internal/benthos/transformers/sha256hash.go @@ -0,0 +1,65 @@ +package neosync_transformers + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/benthosdev/benthos/v4/public/bloblang" + _ "github.com/benthosdev/benthos/v4/public/components/io" + transformer_utils "github.com/nucleuscloud/neosync/worker/internal/benthos/transformers/utils" +) + +func init() { + + spec := bloblang.NewPluginSpec() + + err := bloblang.RegisterFunctionV2("sha256hash", spec, func(args *bloblang.ParsedParams) (bloblang.Function, error) { + + return func() (any, error) { + + val, err := GenerateRandomSHA256Hash() + + if err != nil { + return false, fmt.Errorf("unable to generate sha256 hash") + } + return val, nil + }, nil + }) + if err != nil { + panic(err) + } +} + +func GenerateRandomSHA256Hash() (string, error) { + + length, err := transformer_utils.GenerateRandomInt(1) + if err != nil { + return "", err + } + + str, err := transformer_utils.GenerateRandomStringWithLength(length) + if err != nil { + return "", err + } + + // hash the value + bites := []byte(str) + hasher := sha256.New() + _, err = hasher.Write(bites) + if err != nil { + return "", err + } + + // compute sha256 checksum and encode it into a hex string + hashed := hasher.Sum(nil) + var buf bytes.Buffer + e := hex.NewEncoder(&buf) + _, err = e.Write(hashed) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/worker/internal/benthos/transformers/sha256hash_test.go b/worker/internal/benthos/transformers/sha256hash_test.go new file mode 100644 index 0000000000..51fb40c9b6 --- /dev/null +++ b/worker/internal/benthos/transformers/sha256hash_test.go @@ -0,0 +1,15 @@ +package neosync_transformers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_GenerateSHA256Hash(t *testing.T) { + + res, err := GenerateRandomSHA256Hash() + assert.NoError(t, err) + + assert.IsType(t, "", res) +} diff --git a/worker/pkg/workflows/datasync/activities/activities.go b/worker/pkg/workflows/datasync/activities/activities.go index decb188566..51e1db5df0 100644 --- a/worker/pkg/workflows/datasync/activities/activities.go +++ b/worker/pkg/workflows/datasync/activities/activities.go @@ -598,7 +598,7 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping) (string, error) { case "random_string": pl := col.Transformer.Config.GetRandomStringConfig().PreserveLength sl := col.Transformer.Config.GetRandomStringConfig().StrLength - return fmt.Sprintf(`this.%s.randomstringtransformer(%t, %d)`, col.Column, pl, sl), nil + return fmt.Sprintf(`randomstringtransformer(this.%s,%t, %d)`, col.Column, pl, sl), nil case "random_int": pl := col.Transformer.Config.GetRandomIntConfig().PreserveLength sl := col.Transformer.Config.GetRandomIntConfig().IntLength @@ -629,7 +629,11 @@ func computeMutationFunction(col *mgmtv1alpha1.JobMapping) (string, error) { luhn := col.Transformer.Config.GetCardNumberConfig().ValidLuhn return fmt.Sprintf(`cardnumbertransformer(%t)`, luhn), nil case "sha256": - return fmt.Sprintf(`this.%s.bytes().hash("sha256").encode("hex")`, col.Column), nil + if col.Column != "" { + return fmt.Sprintf(`root = this.%s.bytes().hash("sha256").encode("hex")`, col.Column), nil + } else { + return `sha256hash()`, nil + } case "social_security_number": return "ssntransformer()", nil default: