Merge pull request #21 from zilliztech/test

support vector type for elastic search
zilliztech · Nov 7, 2024 · 8aa6c9c · 8aa6c9c
2 parents 78120e5 + 863a1ac
commit 8aa6c9c
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/...ache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchTypeConverter.java b/...ache/seatunnel/connectors/seatunnel/elasticsearch/catalog/ElasticSearchTypeConverter.java
@@ -72,6 +72,7 @@
 import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.TEXT;
 import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.TOKEN_COUNT;
 import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.UNSIGNED_LONG;
+import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.VECTOR;
 import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.VERSION;
 
 @AutoService(TypeConverter.class)
@@ -105,6 +106,7 @@ public Column convert(BasicTypeDefine<EsType> typeDefine) {
                                         .toArray(SeaTunnelDataType<?>[]::new)));
                 break;
             case DENSE_VECTOR:
+            case VECTOR:
                 String elementType =
                         typeDefine.getNativeType().getOptions().get("element_type").toString();
                 if (elementType.equals("byte")) {

diff --git a/...ain/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/EsRestClient.java b/...ain/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/EsRestClient.java
@@ -20,6 +20,7 @@
 import org.apache.http.Header;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.message.BasicHeader;
+import static org.apache.seatunnel.connectors.seatunnel.elasticsearch.client.EsType.VECTOR;
 import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ArrayNode;
@@ -666,6 +667,19 @@ private static Map<String, BasicTypeDefine<EsType>> getFieldTypeMappingFromPrope
                                     // es dense_vector dim
                                     typeDefine.scale(fieldProperty.get("dims").asInt());
 
+                                } else if (type.equalsIgnoreCase(VECTOR)) {
+                                    //adapt for huawei cloud elastic search
+                                    String elementType =
+                                            fieldProperty.get("dim_type") == null
+                                                    ? "float"
+                                                    : fieldProperty.get("dim_type").asText();
+                                    Map<String, Object> options = new HashMap<>();
+                                    options.put("element_type", elementType);
+                                    typeDefine.nativeType(new EsType(type, options));
+
+                                    // es dense_vector dim
+                                    typeDefine.scale(fieldProperty.get("dimension").asInt());
+
                                 } else if (type.equalsIgnoreCase(DATE)
                                         || type.equalsIgnoreCase(DATE_NANOS)) {
                                     String format =

diff --git a/.../src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/EsType.java b/.../src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/client/EsType.java
@@ -35,6 +35,7 @@ public class EsType {
     public static final String DATE = "date";
     public static final String DATE_NANOS = "date_nanos";
     public static final String DENSE_VECTOR = "dense_vector";
+    public static final String VECTOR = "vector";
     public static final String DOUBLE = "double";
     public static final String FLATTENED = "flattened";
     public static final String FLOAT = "float";