Upgrade AWS SDK Java from 1.x to 2.x (#175)

* Prepare for 5.0.0-snapshot development * Upgrade AWS SDK from 1.x to 2.x * Fix UTs * Fix UTs * Fix UT * Fix UT * Fix null check of collection fields in AttributeValue * Add AttributeValue custom Ser/Des and fix UT * Include dependencies into emr-dynamodb-hadoop JAR * Fix integration gst * Fix UT DynamoDBObjectInspectorTest.testItem * Clean up comments
awslabs · Mar 15, 2023 · 1dec9d1 · 1dec9d1
1 parent e1c937f
commit 1dec9d1
Show file tree

Hide file tree

Showing 77 changed files with 1,197 additions and 795 deletions.
diff --git a/emr-dynamodb-hadoop/pom.xml b/emr-dynamodb-hadoop/pom.xml
@@ -54,8 +54,13 @@
         </dependency>
 
         <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-dynamodb</artifactId>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>dynamodb</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
         </dependency>
 
         <dependency>
@@ -89,6 +94,13 @@
                     <maxAllowedViolations>0</maxAllowedViolations>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/emr-dynamodb-hadoop/src/main/java/org/apache/hadoop/dynamodb/AttributeValueDeserializer.java b/emr-dynamodb-hadoop/src/main/java/org/apache/hadoop/dynamodb/AttributeValueDeserializer.java
@@ -0,0 +1,120 @@
+/**
+ * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file
+ * except in compliance with the License. A copy of the License is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "LICENSE.TXT" file accompanying this file. This file is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.dynamodb;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.dynamodb.type.DynamoDBTypeConstants;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+
+public class AttributeValueDeserializer implements JsonDeserializer<AttributeValue> {
+
+  @Override
+  public AttributeValue deserialize(JsonElement jsonElement, Type type,
+                                    JsonDeserializationContext context) throws JsonParseException {
+    if (jsonElement.isJsonNull()) {
+      return null;
+    }
+
+    if (jsonElement.isJsonObject()) {
+      JsonObject jsonObject = jsonElement.getAsJsonObject();
+
+      for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
+        String attributeName = entry.getKey();
+        JsonElement attributeValue = entry.getValue();
+
+        if (DynamoDBTypeConstants.BINARY.equalsIgnoreCase(attributeName)) {
+          return AttributeValue.fromB(context.deserialize(attributeValue, SdkBytes.class));
+        }
+
+        if (DynamoDBTypeConstants.BINARY_SET.equalsIgnoreCase(attributeName)) {
+          JsonArray jsonArray = attributeValue.getAsJsonArray();
+          if (!jsonArray.isEmpty()) {
+            List<SdkBytes> sdkBytesList = new ArrayList<>();
+            jsonArray.forEach(item -> sdkBytesList.add(context.deserialize(item, SdkBytes.class)));
+            return AttributeValue.fromBs(sdkBytesList);
+          }
+        }
+
+        if (DynamoDBTypeConstants.BOOLEAN.equalsIgnoreCase(attributeName)) {
+          return AttributeValue.fromBool(attributeValue.getAsBoolean());
+        }
+
+        if (DynamoDBTypeConstants.NULL.equalsIgnoreCase(attributeName)) {
+          return AttributeValue.fromNul(attributeValue.getAsBoolean());
+        }
+
+        if (DynamoDBTypeConstants.NUMBER.equalsIgnoreCase(attributeName)) {
+          return AttributeValue.fromN(attributeValue.getAsString());
+        }
+
+        if (DynamoDBTypeConstants.NUMBER_SET.equalsIgnoreCase(attributeName)) {
+          JsonArray jsonArray = attributeValue.getAsJsonArray();
+          if (!jsonArray.isEmpty()) {
+            List<String> numberList = new ArrayList<>();
+            jsonArray.forEach(item -> numberList.add(item.getAsString()));
+            return AttributeValue.fromNs(numberList);
+          }
+        }
+
+        if (DynamoDBTypeConstants.LIST.equalsIgnoreCase(attributeName)) {
+          JsonArray jsonArray = attributeValue.getAsJsonArray();
+          if (!jsonArray.isEmpty()) {
+            List<AttributeValue> avl = new ArrayList<>();
+            jsonArray.forEach(element ->
+                avl.add(context.deserialize(element, AttributeValue.class)));
+            return AttributeValue.fromL(avl);
+          }
+        }
+
+        if (DynamoDBTypeConstants.MAP.equalsIgnoreCase(attributeName)) {
+          JsonObject jsonMap = attributeValue.getAsJsonObject();
+          if (jsonMap.size() != 0) {
+            Map<String, AttributeValue> avm = new HashMap<>();
+            jsonMap.entrySet().forEach(item -> {
+              avm.put(item.getKey(), context.deserialize(item.getValue(), AttributeValue.class));
+            });
+            return AttributeValue.fromM(avm);
+          }
+        }
+
+        if (DynamoDBTypeConstants.STRING.equalsIgnoreCase(attributeName)) {
+          return AttributeValue.fromS(attributeValue.getAsString());
+        }
+
+        if (DynamoDBTypeConstants.STRING_SET.equalsIgnoreCase(attributeName)) {
+          JsonArray jsonArray = attributeValue.getAsJsonArray();
+          if (!jsonArray.isEmpty()) {
+            List<String> stringList = new ArrayList<>();
+            jsonArray.forEach(item -> stringList.add(item.getAsString()));
+            return AttributeValue.fromSs(stringList);
+          }
+        }
+      }
+    }
+
+    // Return an empty instance as default value.
+    return AttributeValue.builder().build();
+  }
+}
diff --git a/emr-dynamodb-hadoop/src/main/java/org/apache/hadoop/dynamodb/AttributeValueSerializer.java b/emr-dynamodb-hadoop/src/main/java/org/apache/hadoop/dynamodb/AttributeValueSerializer.java
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file
+ * except in compliance with the License. A copy of the License is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "LICENSE.TXT" file accompanying this file. This file is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.dynamodb;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import java.lang.reflect.Type;
+import org.apache.hadoop.dynamodb.type.DynamoDBTypeConstants;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+
+public class AttributeValueSerializer implements JsonSerializer<AttributeValue> {
+
+  @Override
+  public JsonElement serialize(AttributeValue attributeValue, Type type,
+                               JsonSerializationContext context) {
+    if (attributeValue == null) {
+      return JsonNull.INSTANCE;
+    }
+
+    JsonObject serializedValue = new JsonObject();
+    switch (attributeValue.type()) {
+      case B:
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.BINARY),
+            context.serialize(attributeValue.b()));
+        break;
+      case BOOL:
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.BOOLEAN),
+            new JsonPrimitive(attributeValue.bool()));
+        break;
+      case BS:
+        JsonArray sdkBytesList = new JsonArray();
+        attributeValue.bs()
+            .forEach(item -> sdkBytesList.add(context.serialize(item, SdkBytes.class)));
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.BINARY_SET), sdkBytesList);
+        break;
+      case L:
+        JsonArray attributeList = new JsonArray();
+        attributeValue.l()
+            .forEach(item -> attributeList.add(context.serialize(item, AttributeValue.class)));
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.LIST), attributeList);
+        break;
+      case M:
+        JsonObject avm = new JsonObject();
+        attributeValue.m().entrySet()
+            .forEach(entry ->
+                avm.add(entry.getKey(), context.serialize(entry.getValue(), AttributeValue.class)));
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.MAP), avm);
+        break;
+      case N:
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.NUMBER),
+            new JsonPrimitive(attributeValue.n()));
+        break;
+      case NS:
+        JsonArray numberList = new JsonArray();
+        attributeValue.ns().forEach(item -> numberList.add(new JsonPrimitive(item)));
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.NUMBER_SET), numberList);
+        break;
+      case NUL:
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.NULL),
+            new JsonPrimitive(attributeValue.nul()));
+        break;
+      case S:
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.STRING),
+            new JsonPrimitive(attributeValue.s()));
+        break;
+      case SS:
+        JsonArray stringList = new JsonArray();
+        attributeValue.ss().forEach(item -> stringList.add(new JsonPrimitive(item)));
+        serializedValue.add(toV1FieldCasingStyle(DynamoDBTypeConstants.STRING_SET), stringList);
+        break;
+      default:
+        break;
+    }
+    return serializedValue;
+  }
+
+  private static String toV1FieldCasingStyle(String typeConstant) {
+    return typeConstant.substring(0, 1).toLowerCase() + typeConstant.substring(1).toUpperCase();
+  }
+}