fix issue with decimals losing precision

Altinity · Oct 25, 2024 · 4629a01 · 4629a01
1 parent 5087a57
commit 4629a01
Showing 1 changed file with 8 additions and 3 deletions.
diff --git a/parquetify/src/main/java/GenerateParquet.java b/parquetify/src/main/java/GenerateParquet.java
@@ -10,6 +10,7 @@
 import org.apache.parquet.hadoop.example.GroupWriteSupport;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.apache.parquet.hadoop.util.HadoopOutputFile;
+import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
@@ -375,6 +376,7 @@ private static void insertDataIntoGroup(Group group, JSONArray schemaArray, int
         }
     }
 
+
     private static void appendValueToGroup(Group group, String name, Object value) {
         try {
             if (value instanceof Integer) {
@@ -385,7 +387,7 @@ private static void appendValueToGroup(Group group, String name, Object value) {
                 // Handle UUID string by converting to 16-byte array if the field is UUID
                 if (isUUID(name)) {
                     byte[] uuidBytes = hexStringToByteArray((String) value);
-                    group.add(name, org.apache.parquet.io.api.Binary.fromConstantByteArray(uuidBytes));
+                    group.add(name, Binary.fromConstantByteArray(uuidBytes));
                 } else {
                     group.add(name, (String) value);
                 }
@@ -396,9 +398,11 @@ private static void appendValueToGroup(Group group, String name, Object value) {
             } else if (value instanceof Float) {
                 group.add(name, (Float) value);
             } else if (value instanceof BigDecimal) {
-                group.add(name, ((BigDecimal) value).doubleValue());
+                BigDecimal decimalValue = (BigDecimal) value;
+                byte[] bytes = decimalValue.unscaledValue().toByteArray();
+                group.add(name, Binary.fromConstantByteArray(bytes));
             } else if (value instanceof byte[]) {
-                group.add(name, org.apache.parquet.io.api.Binary.fromConstantByteArray((byte[]) value));
+                group.add(name, Binary.fromConstantByteArray((byte[]) value));
             } else if (value instanceof JSONObject) {
                 Group nestedGroup = group.addGroup(name);
                 JSONObject jsonObject = (JSONObject) value;
@@ -414,6 +418,7 @@ private static void appendValueToGroup(Group group, String name, Object value) {
         }
     }
 
+
     private static boolean isUUID(String fieldName) {
         // We add logic to identify if a particular field is a UUID
         // For simplicity, let's assume field names ending with '_uuid' indicate UUID