diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd9951ff..5cf02d71b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,16 @@ -# Changelog +# Changelog + +## milvus-sdk-java 2.0.1 (2021-01-18) + +### Improvement + +- \#248 - Pass travel timestamp and guarantee timestamp for query/search interface + +## milvus-sdk-java 2.0.0 (2021-12-31) + +### Feature + +- \#183 - java sdk for milvus 2.0 ## milvus-sdk-java 0.8.5 (2020-08-26) diff --git a/README.md b/README.md index 1369432ab..9e42b1448 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following table shows compatibilities between Milvus and Java SDK. | Milvus version | Java SDK version | | :------------: | :--------------: | -| 2.0 | 2.0.0 | +| 2.0 | 2.0.1 | ### Install Java SDK @@ -27,14 +27,14 @@ You can use **Apache Maven** or **Gradle**/**Grails** to download the SDK. io.milvus milvus-sdk-java - 2.0.0 + 2.0.1 ``` - Gradle/Grails ```gradle - compile 'io.milvus:milvus-sdk-java:2.0.0' + compile 'io.milvus:milvus-sdk-java:2.0.1' ``` ### Examples diff --git a/examples/main/io/milvus/GeneralExample.java b/examples/main/io/milvus/GeneralExample.java index 540469360..6c77d96aa 100644 --- a/examples/main/io/milvus/GeneralExample.java +++ b/examples/main/io/milvus/GeneralExample.java @@ -70,7 +70,7 @@ private void handleResponseStatus(R r) { } } - private R createCollection(long timeoutMiliseconds) { + private R createCollection(long timeoutMilliseconds) { System.out.println("========== createCollection() =========="); FieldType fieldType1 = FieldType.newBuilder() .withName(ID_FIELD) @@ -109,7 +109,7 @@ private R createCollection(long timeoutMiliseconds) { .addFieldType(fieldType3) // .addFieldType(fieldType4) .build(); - R response = milvusClient.withTimeout(timeoutMiliseconds, TimeUnit.MILLISECONDS) + R response = milvusClient.withTimeout(timeoutMilliseconds, TimeUnit.MILLISECONDS) .createCollection(createCollectionReq); handleResponseStatus(response); System.out.println(response); @@ -315,6 +315,7 @@ private R delete(String partitionName, String expr) { private R searchFace(String expr) { System.out.println("========== searchFace() =========="); + long begin = System.currentTimeMillis(); List outFields = Collections.singletonList(AGE_FIELD); List> vectors = generateFloatVectors(5); @@ -328,9 +329,14 @@ private R searchFace(String expr) { .withVectorFieldName(VECTOR_FIELD) .withExpr(expr) .withParams(SEARCH_PARAM) + .withGuaranteeTimestamp(Constant.GUARANTEE_EVENTUALLY_TS) .build(); R response = milvusClient.search(searchParam); + long end = System.currentTimeMillis(); + long cost = (end - begin); + System.out.println("Search time cost: " + cost + "ms"); + handleResponseStatus(response); SearchResultsWrapper wrapper = new SearchResultsWrapper(response.getData().getResults()); for (int i = 0; i < vectors.size(); ++i) { @@ -346,6 +352,7 @@ private R searchFace(String expr) { // private R searchProfile(String expr) { // System.out.println("========== searchProfile() =========="); +// long begin = System.currentTimeMillis(); // // List outFields = Collections.singletonList(AGE_FIELD); // List vectors = generateBinaryVectors(5); @@ -363,6 +370,10 @@ private R searchFace(String expr) { // // // R response = milvusClient.search(searchParam); +// long end = System.currentTimeMillis(); +// long cost = (end - begin); +// System.out.println("Search time cost: " + cost + "ms"); +// // handleResponseStatus(response); // SearchResultsWrapper wrapper = new SearchResultsWrapper(response.getData().getResults()); // for (int i = 0; i < vectors.size(); ++i) { diff --git a/examples/pom.xml b/examples/pom.xml index 940616984..6b3f6e4aa 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -25,7 +25,7 @@ io.milvus milvus-sdk-java-examples - 2.0.0 + 2.0.1 @@ -63,7 +63,7 @@ io.milvus milvus-sdk-java - 2.0.0 + 2.0.1 com.google.code.gson diff --git a/pom.xml b/pom.xml index aa76fc727..3c5f2387e 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ io.milvus milvus-sdk-java - 2.0.0 + 2.0.1 jar io.milvus:milvus-sdk-java diff --git a/src/main/java/io/milvus/Response/MutationResultWrapper.java b/src/main/java/io/milvus/Response/MutationResultWrapper.java index ce921bc6f..6c9ec9587 100644 --- a/src/main/java/io/milvus/Response/MutationResultWrapper.java +++ b/src/main/java/io/milvus/Response/MutationResultWrapper.java @@ -63,4 +63,16 @@ public List getStringIDs() throws ParamException { public long getDeleteCount() { return result.getDeleteCnt(); } + + /** + * Get timestamp of the operation marked by server. You can use this timestamp as for guarantee timestamp of query/search api. + * + * Note: the timestamp is not an absolute timestamp, it is a hybrid value combined by UTC time and internal flags. + * We call it TSO, for more information please refer to: https://github.com/milvus-io/milvus/blob/master/docs/design_docs/milvus_hybrid_ts_en.md + * + * @return int row count of the deleted entities + */ + public long getOperationTs() { + return result.getTimestamp(); + } } diff --git a/src/main/java/io/milvus/client/AbstractMilvusGrpcClient.java b/src/main/java/io/milvus/client/AbstractMilvusGrpcClient.java index 71ed1b8cf..487bf763c 100644 --- a/src/main/java/io/milvus/client/AbstractMilvusGrpcClient.java +++ b/src/main/java/io/milvus/client/AbstractMilvusGrpcClient.java @@ -1462,6 +1462,9 @@ public R search(@NonNull SearchParam requestParam) { builder.setDsl(requestParam.getExpr()); } + builder.setTravelTimestamp(requestParam.getTravelTimestamp()); + builder.setGuaranteeTimestamp(requestParam.getGuaranteeTimestamp()); + SearchRequest searchRequest = builder.build(); SearchResults response = this.blockingStub().search(searchRequest); @@ -1499,6 +1502,8 @@ public R query(@NonNull QueryParam requestParam) { .addAllPartitionNames(requestParam.getPartitionNames()) .addAllOutputFields(requestParam.getOutFields()) .setExpr(requestParam.getExpr()) + .setTravelTimestamp(requestParam.getTravelTimestamp()) + .setGuaranteeTimestamp(requestParam.getGuaranteeTimestamp()) .build(); QueryResults response = this.blockingStub().query(queryRequest); diff --git a/src/main/java/io/milvus/param/Constant.java b/src/main/java/io/milvus/param/Constant.java index 390e9a5dc..d66c0ffc8 100644 --- a/src/main/java/io/milvus/param/Constant.java +++ b/src/main/java/io/milvus/param/Constant.java @@ -48,4 +48,13 @@ public class Constant { // max value for waiting create index interval, unit: millisecond public static final Long MAX_WAITING_INDEX_INTERVAL = 2000L; + + + // set this value for "withGuaranteeTimestamp" of QueryParam/SearchParam + // to instruct server execute query/search immediately. + public static final Long GUARANTEE_EVENTUALLY_TS = 1L; + + // set this value for "withGuaranteeTimestamp" of QueryParam/SearchParam + // to instruct server execute query/search after all DML operations finished. + public static final Long GUARANTEE_STRONG_TS = 0L; } diff --git a/src/main/java/io/milvus/param/dml/QueryParam.java b/src/main/java/io/milvus/param/dml/QueryParam.java index 8b051de46..b1c7a4b23 100644 --- a/src/main/java/io/milvus/param/dml/QueryParam.java +++ b/src/main/java/io/milvus/param/dml/QueryParam.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import io.milvus.exception.ParamException; +import io.milvus.param.Constant; import io.milvus.param.ParamUtils; import lombok.Getter; import lombok.NonNull; @@ -37,12 +38,16 @@ public class QueryParam { private final List partitionNames; private final List outFields; private final String expr; + private final long travelTimestamp; + private final long guaranteeTimestamp; private QueryParam(@NonNull Builder builder) { this.collectionName = builder.collectionName; this.partitionNames = builder.partitionNames; this.outFields = builder.outFields; this.expr = builder.expr; + this.travelTimestamp = builder.travelTimestamp; + this.guaranteeTimestamp = builder.guaranteeTimestamp; } public static Builder newBuilder() { @@ -57,6 +62,8 @@ public static class Builder { private final List partitionNames = Lists.newArrayList(); private final List outFields = new ArrayList<>(); private String expr = ""; + private Long travelTimestamp = 0L; + private Long guaranteeTimestamp = Constant.GUARANTEE_EVENTUALLY_TS; private Builder() { } @@ -132,6 +139,37 @@ public Builder withExpr(@NonNull String expr) { return this; } + /** + * Specify an absolute timestamp in a query to get results based on a data view at a specified point in time. + * Default value is 0, server executes query on a full data view. + * + * @param ts a timestamp value + * @return Builder + */ + public Builder withTravelTimestamp(@NonNull Long ts) { + this.travelTimestamp = ts; + return this; + } + + /** + * Instructs server to see insert/delete operations performed before a provided timestamp. + * If no such timestamp is specified, the server will wait for the latest operation to finish and query. + * + * Note: The timestamp is not an absolute timestamp, it is a hybrid value combined by UTC time and internal flags. + * We call it TSO, for more information please refer to: https://github.com/milvus-io/milvus/blob/master/docs/design_docs/milvus_hybrid_ts_en.md + * You can get a TSO from insert/delete operations, see the MutationResultWrapper class. + * Use an operation's TSO to set this parameter, the server will execute query after this operation is finished. + * + * Default value is GUARANTEE_EVENTUALLY_TS, query executes query immediately. + * + * @param ts a timestamp value + * @return Builder + */ + public Builder withGuaranteeTimestamp(@NonNull Long ts) { + this.guaranteeTimestamp = ts; + return this; + } + /** * Verifies parameters and creates a new QueryParam instance. * @@ -141,6 +179,14 @@ public QueryParam build() throws ParamException { ParamUtils.CheckNullEmptyString(collectionName, "Collection name"); ParamUtils.CheckNullEmptyString(expr, "Expression"); + if (travelTimestamp < 0) { + throw new ParamException("The travel timestamp must be greater than 0"); + } + + if (guaranteeTimestamp < 0) { + throw new ParamException("The guarantee timestamp must be greater than 0"); + } + return new QueryParam(this); } } diff --git a/src/main/java/io/milvus/param/dml/SearchParam.java b/src/main/java/io/milvus/param/dml/SearchParam.java index 7a92cb131..56081a8f5 100644 --- a/src/main/java/io/milvus/param/dml/SearchParam.java +++ b/src/main/java/io/milvus/param/dml/SearchParam.java @@ -21,13 +21,13 @@ import com.google.common.collect.Lists; import io.milvus.exception.ParamException; +import io.milvus.param.Constant; import io.milvus.param.MetricType; import io.milvus.param.ParamUtils; import lombok.Getter; import lombok.NonNull; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; /** @@ -45,6 +45,8 @@ public class SearchParam { private final List vectors; private final int roundDecimal; private final String params; + private final long travelTimestamp; + private final long guaranteeTimestamp; private SearchParam(@NonNull Builder builder) { this.collectionName = builder.collectionName; @@ -57,6 +59,8 @@ private SearchParam(@NonNull Builder builder) { this.vectors = builder.vectors; this.roundDecimal = builder.roundDecimal; this.params = builder.params; + this.travelTimestamp = builder.travelTimestamp; + this.guaranteeTimestamp = builder.guaranteeTimestamp; } public static Builder newBuilder() { @@ -73,10 +77,12 @@ public static class Builder { private String vectorFieldName; private Integer topK; private String expr = ""; - private List outFields = new ArrayList<>(); + private final List outFields = Lists.newArrayList(); private List vectors; private Integer roundDecimal = -1; private String params = "{}"; + private Long travelTimestamp = 0L; + private Long guaranteeTimestamp = Constant.GUARANTEE_EVENTUALLY_TS; Builder() { } @@ -168,7 +174,7 @@ public Builder withExpr(@NonNull String expr) { * @return Builder */ public Builder withOutFields(@NonNull List outFields) { - this.outFields = outFields; + outFields.forEach(this::addOutField); return this; } @@ -223,6 +229,37 @@ public Builder withParams(@NonNull String params) { return this; } + /** + * Specify an absolute timestamp in a search to get results based on a data view at a specified point in time. + * Default value is 0, server executes search on a full data view. + * + * @param ts a timestamp value + * @return Builder + */ + public Builder withTravelTimestamp(@NonNull Long ts) { + this.travelTimestamp = ts; + return this; + } + + /** + * Instructs server to see insert/delete operations performed before a provided timestamp. + * If no such timestamp is specified, the server will wait for the latest operation to finish and search. + * + * Note: The timestamp is not an absolute timestamp, it is a hybrid value combined by UTC time and internal flags. + * We call it TSO, for more information please refer to: https://github.com/milvus-io/milvus/blob/master/docs/design_docs/milvus_hybrid_ts_en.md + * You can get a TSO from insert/delete operations, see the MutationResultWrapper class. + * Use an operation's TSO to set this parameter, the server will execute search after this operation is finished. + * + * Default value is GUARANTEE_EVENTUALLY_TS, server executes search immediately. + * + * @param ts a timestamp value + * @return Builder + */ + public Builder withGuaranteeTimestamp(@NonNull Long ts) { + this.guaranteeTimestamp = ts; + return this; + } + /** * Verifies parameters and creates a new SearchParam instance. * @@ -236,6 +273,14 @@ public SearchParam build() throws ParamException { throw new ParamException("TopK value is illegal"); } + if (travelTimestamp < 0) { + throw new ParamException("The travel timestamp must be greater than 0"); + } + + if (guaranteeTimestamp < 0) { + throw new ParamException("The guarantee timestamp must be greater than 0"); + } + if (metricType == MetricType.INVALID) { throw new ParamException("Metric type is illegal"); } diff --git a/src/main/java/io/milvus/param/index/CreateIndexParam.java b/src/main/java/io/milvus/param/index/CreateIndexParam.java index 49cc58b8b..3fe142c2f 100644 --- a/src/main/java/io/milvus/param/index/CreateIndexParam.java +++ b/src/main/java/io/milvus/param/index/CreateIndexParam.java @@ -212,7 +212,7 @@ public CreateIndexParam build() throws ParamException { } } - ParamUtils.CheckNullEmptyString(extraParam, "Index extra param"); +// ParamUtils.CheckNullEmptyString(extraParam, "Index extra param"); return new CreateIndexParam(this); } diff --git a/src/test/java/io/milvus/client/MilvusServiceClientTest.java b/src/test/java/io/milvus/client/MilvusServiceClientTest.java index b85081575..799ceea47 100644 --- a/src/test/java/io/milvus/client/MilvusServiceClientTest.java +++ b/src/test/java/io/milvus/client/MilvusServiceClientTest.java @@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertFalse; class MilvusServiceClientTest { private final int testPort = 53019; @@ -1092,7 +1091,28 @@ void createIndexParam() { .withFieldName("field1") .withIndexType(IndexType.IVF_FLAT) .withMetricType(MetricType.L2) - .withExtraParam("") + .withSyncMode(Boolean.TRUE) + .withSyncWaitingInterval(-1L) + .build() + ); + + assertThrows(ParamException.class, () -> CreateIndexParam.newBuilder() + .withCollectionName("collection1") + .withFieldName("field1") + .withIndexType(IndexType.IVF_FLAT) + .withMetricType(MetricType.L2) + .withSyncMode(Boolean.TRUE) + .withSyncWaitingInterval(Constant.MAX_WAITING_INDEX_INTERVAL + 1L) + .build() + ); + + assertThrows(ParamException.class, () -> CreateIndexParam.newBuilder() + .withCollectionName("collection1") + .withFieldName("field1") + .withIndexType(IndexType.IVF_FLAT) + .withMetricType(MetricType.L2) + .withSyncMode(Boolean.TRUE) + .withSyncWaitingTimeout(0L) .build() ); } @@ -1461,6 +1481,36 @@ void searchParam() { .build() ); + assertThrows(ParamException.class, () -> SearchParam.newBuilder() + .withCollectionName("collection1") + .withPartitionNames(partitions) + .addPartitionName("p2") + .withParams("dummy") + .withOutFields(outputFields) + .withVectorFieldName("field1") + .withMetricType(MetricType.IP) + .withTopK(5) + .withVectors(vectors) + .withExpr("dummy") + .withTravelTimestamp(-1L) + .build() + ); + + assertThrows(ParamException.class, () -> SearchParam.newBuilder() + .withCollectionName("collection1") + .withPartitionNames(partitions) + .addPartitionName("p2") + .withParams("dummy") + .withOutFields(outputFields) + .withVectorFieldName("field1") + .withMetricType(MetricType.IP) + .withTopK(5) + .withVectors(vectors) + .withExpr("dummy") + .withGuaranteeTimestamp(-1L) + .build() + ); + List vector1 = Collections.singletonList(0.1F); vectors.add(vector1); assertThrows(ParamException.class, () -> SearchParam.newBuilder() @@ -1598,12 +1648,15 @@ void search() { .withPartitionNames(partitions) .withParams("dummy") .withOutFields(outputFields) + .addOutField("f2") .withVectorFieldName("field1") .withMetricType(MetricType.IP) .withTopK(5) .withVectors(vectors) .withExpr("dummy") .withRoundDecimal(5) + .withTravelTimestamp(1L) + .withGuaranteeTimestamp(1L) .build(); R resp = client.search(param); assertEquals(R.Status.Success.getCode(), resp.getStatus()); @@ -1663,6 +1716,24 @@ void queryParam() { .withExpr("") .build() ); + + assertThrows(ParamException.class, () -> QueryParam.newBuilder() + .withCollectionName("collection1") + .withPartitionNames(partitions) + .withOutFields(outputFields) + .withExpr("dummy") + .withTravelTimestamp(-1L) + .build() + ); + + assertThrows(ParamException.class, () -> QueryParam.newBuilder() + .withCollectionName("collection1") + .withPartitionNames(partitions) + .withOutFields(outputFields) + .withExpr("dummy") + .withGuaranteeTimestamp(-1L) + .build() + ); } @Test @@ -1673,7 +1744,10 @@ void query() { .withCollectionName("collection1") .withPartitionNames(partitions) .withOutFields(outputFields) + .addOutField("d1") .withExpr("dummy") + .withTravelTimestamp(0L) + .withGuaranteeTimestamp(1L) .build(); testFuncByName("query", param); @@ -2157,8 +2231,10 @@ void testMutationResultWrapper() { .setIntId(LongArray.newBuilder() .addAllData(nID) .build())) + .setTimestamp(1000) .build(); MutationResultWrapper longWrapper = new MutationResultWrapper(results); + assertEquals(1000, longWrapper.getOperationTs()); assertEquals(nID.size(), longWrapper.getInsertCount()); assertEquals(nID.size(), longWrapper.getDeleteCount()); assertThrows(ParamException.class, longWrapper::getStringIDs);