Skip to content

Commit a301439

Browse files
rohanshah18rohanshah18
authored
Revert "Refactor dataplane " (#67)
Reverts #66 to release serverless changes for control plane operations.
1 parent 26a1348 commit a301439

File tree

57 files changed

+490
-27171
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+490
-27171
lines changed

‎build.gradle‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
plugins {
22
id 'com.github.johnrengelman.shadow' version '6.1.0'
33
id 'java-library'
4+
id 'com.google.protobuf' version '0.8.18'
45
id 'maven-publish'
56
id 'signing'
67
id 'idea'
@@ -52,7 +53,6 @@ dependencies {
5253
implementation 'com.squareup.okhttp3:logging-interceptor:4.10.0'
5354
implementation 'io.gsonfire:gson-fire:1.8.5'
5455
implementation 'org.openapitools:jackson-databind-nullable:0.2.6'
55-
implementation 'com.google.protobuf:protobuf-java:3.25.2'
5656
compileOnly "org.apache.tomcat:annotations-api:6.0.53" // necessary for Java 9+
5757

5858
testImplementation("com.squareup.okhttp3:mockwebserver:4.10.0")
@@ -113,6 +113,22 @@ task relocateShadowJar(type: ConfigureShadowRelocation) {
113113

114114
tasks.shadowJar.dependsOn tasks.relocateShadowJar
115115

116+
protobuf {
117+
protoc {
118+
artifact = "com.google.protobuf:protoc:3.19.4"
119+
}
120+
plugins {
121+
grpc {
122+
artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}"
123+
}
124+
}
125+
generateProtoTasks {
126+
all()*.plugins {
127+
grpc {}
128+
}
129+
}
130+
}
131+
116132
publishing {
117133
publications {
118134
pineconeClientMaven(MavenPublication) {

‎src/integration/java/io/pinecone/integration/controlPlane/pod/ConfigureIndexTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io.pinecone.PineconeControlPlaneClient;
44
import io.pinecone.exceptions.PineconeException;
55
import io.pinecone.exceptions.PineconeForbiddenException;
6+
import io.pinecone.integration.dataplane.PineconeClientLiveIntegTest;
67
import io.pinecone.exceptions.PineconeBadRequestException;
78
import io.pinecone.exceptions.PineconeNotFoundException;
89
import org.junit.jupiter.api.*;

‎src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
package io.pinecone.integration.dataplane;
2+
3+
import com.google.common.primitives.Floats;
4+
import com.google.protobuf.Struct;
5+
import com.google.protobuf.Value;
6+
import io.pinecone.PineconeConnection;
7+
import io.pinecone.helpers.RandomStringBuilder;
8+
import io.pinecone.proto.*;
9+
import org.junit.jupiter.api.AfterAll;
10+
import org.junit.jupiter.api.Assertions;
11+
import org.junit.jupiter.api.BeforeAll;
12+
import org.junit.jupiter.api.Test;
13+
import org.openapitools.client.model.IndexModelSpec;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
16+
17+
import java.io.IOException;
18+
import java.util.ArrayList;
19+
import java.util.Arrays;
20+
import java.util.List;
21+
22+
import static io.pinecone.helpers.IndexManager.createIndexIfNotExistsDataPlane;
23+
import static io.pinecone.helpers.AssertRetry.assertWithRetry;
24+
25+
public class PineconeClientLiveIntegTest {
26+
27+
private static final Logger logger = LoggerFactory.getLogger(PineconeClientLiveIntegTest.class);
28+
29+
private static PineconeConnection connection;
30+
private static VectorServiceGrpc.VectorServiceBlockingStub blockingStub;
31+
32+
@BeforeAll
33+
public static void defineConfig() throws IOException, InterruptedException {
34+
connection = createIndexIfNotExistsDataPlane(3, IndexModelSpec.SERIALIZED_NAME_POD);
35+
blockingStub = connection.getBlockingStub();
36+
}
37+
38+
@AfterAll
39+
public static void cleanUp() {
40+
connection.close();
41+
}
42+
43+
@Test
44+
public void sanity() throws InterruptedException {
45+
String namespace = RandomStringBuilder.build("ns", 8);
46+
47+
// upsert
48+
float[][] upsertData = {{1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}, {7.0F, 8.0F, 9.0F}};
49+
List<String> upsertIds = Arrays.asList("v1", "v2", "v3");
50+
List<Vector> upsertVectors = new ArrayList<>();
51+
52+
for (int i = 0; i < upsertData.length; i++) {
53+
upsertVectors.add(Vector.newBuilder()
54+
.addAllValues(Floats.asList(upsertData[i]))
55+
.setMetadata(Struct.newBuilder()
56+
.putFields("some_field", Value.newBuilder().setNumberValue(i).build())
57+
.build())
58+
.setId(upsertIds.get(i))
59+
.build());
60+
}
61+
62+
UpsertRequest request = UpsertRequest.newBuilder()
63+
.addAllVectors(upsertVectors)
64+
.setNamespace(namespace)
65+
.build();
66+
67+
UpsertResponse upsertResponse = blockingStub.upsert(request);
68+
logger.info("Put " + upsertResponse.getUpsertedCount() + " vectors into the index");
69+
assert (upsertResponse.getUpsertedCount() == 3);
70+
71+
// hybrid upsert
72+
List<String> hybridsIds = Arrays.asList("v4", "v5", "v6");
73+
List<Vector> hybridVectors = new ArrayList<>();
74+
List<Integer> sparseIndices = Arrays.asList(0, 1, 2);
75+
List<Float> sparseValues = Arrays.asList(0.11f, 0.22f, 0.33f);
76+
for (int i = 0; i < hybridsIds.size(); i++) {
77+
hybridVectors.add(
78+
Vector.newBuilder()
79+
.addAllValues(Floats.asList(upsertData[i]))
80+
.setSparseValues(
81+
SparseValues.newBuilder().addAllIndices(sparseIndices).addAllValues(sparseValues).build()
82+
)
83+
.setId(hybridsIds.get(i))
84+
.build());
85+
}
86+
87+
UpsertRequest hybridRequest = UpsertRequest.newBuilder()
88+
.addAllVectors(hybridVectors)
89+
.setNamespace(namespace)
90+
.build();
91+
UpsertResponse hybridResponse = blockingStub.upsert(hybridRequest);
92+
logger.info("Put " + hybridResponse.getUpsertedCount() + " vectors into the index");
93+
assert (hybridResponse.getUpsertedCount() == 3);
94+
95+
// fetch
96+
List<String> ids = Arrays.asList("v1", "v2");
97+
FetchRequest fetchRequest1 = FetchRequest.newBuilder().addAllIds(ids).setNamespace(namespace).build();
98+
assertWithRetry(() -> {
99+
FetchResponse fetchResponse = blockingStub.fetch(fetchRequest1);
100+
assert (fetchResponse.containsVectors("v1"));
101+
});
102+
103+
// Updates vector v1's values to 10.0, 11.0, and 12.0 from 1.0, 2.0, and 3.0
104+
List<Float> updateValueList = Floats.asList(10F, 11F, 12F);
105+
UpdateRequest updateRequest = UpdateRequest.newBuilder()
106+
.setId("v1")
107+
.setNamespace(namespace)
108+
.addAllValues(updateValueList)
109+
.build();
110+
blockingStub.update(updateRequest);
111+
112+
// DEPRECATED: queries parameter of QueryRequest has been deprecated
113+
// Use vector parameter and the associated methods.
114+
// Below commented example shows addQueries() which is deprecated
115+
/*
116+
float[] rawVector = {1.0F, 2.0F, 3.0F};
117+
QueryVector queryVector = QueryVector.newBuilder()
118+
.addAllValues(Floats.asList(rawVector))
119+
.setFilter(Struct.newBuilder()
120+
.putFields("some_field", Value.newBuilder()
121+
.setStructValue(Struct.newBuilder()
122+
.putFields("$lt", Value.newBuilder()
123+
.setNumberValue(3)
124+
.build()))
125+
.build())
126+
.build())
127+
.setNamespace(namespace)
128+
.build();
129+
130+
QueryRequest batchQueryRequest = QueryRequest.newBuilder()
131+
.addQueries(queryVector) // Deprecated
132+
.setNamespace(namespace)
133+
.setTopK(2)
134+
.setIncludeMetadata(true)
135+
.build();
136+
*/
137+
138+
// Below example shows usage of addAllVector() which is associated with vector parameter of QueryRequest.
139+
Iterable<Float> iterableVector = Arrays.asList(1.0F, 2.0F, 3.0F);
140+
QueryRequest queryRequest = QueryRequest.newBuilder()
141+
.addAllVector(iterableVector)
142+
.setNamespace(namespace)
143+
.setTopK(2)
144+
.setIncludeMetadata(true)
145+
.build();
146+
147+
// When querying using a single vector, we get matches instead of results
148+
assertWithRetry(() -> {
149+
QueryResponse queryResponse = blockingStub.query(queryRequest);
150+
Assertions.assertEquals(queryResponse.getMatchesCount(), 2);
151+
});
152+
153+
// Query by id example
154+
QueryRequest queryByIdRequest = QueryRequest.newBuilder()
155+
.setId("v1")
156+
.setNamespace(namespace)
157+
.setTopK(1)
158+
.setIncludeMetadata(true)
159+
.setIncludeValues(true)
160+
.build();
161+
162+
assertWithRetry(() -> {
163+
QueryResponse queryResponse = blockingStub.query(queryByIdRequest);
164+
Assertions.assertEquals(queryResponse.getMatchesCount(),1);
165+
assert (queryResponse.getMatches(0).getValuesList().containsAll(updateValueList));
166+
});
167+
168+
// Delete
169+
String[] idsToDelete = {"v2"};
170+
DeleteRequest deleteRequest = DeleteRequest.newBuilder()
171+
.setNamespace(namespace)
172+
.addAllIds(Arrays.asList(idsToDelete))
173+
.setDeleteAll(false)
174+
.build();
175+
176+
blockingStub.delete(deleteRequest);
177+
FetchRequest fetchRequest2 = FetchRequest.newBuilder().addAllIds(ids).setNamespace(namespace).build();
178+
assertWithRetry(() -> {
179+
FetchResponse fetchResponse = blockingStub.fetch(fetchRequest2);
180+
Assertions.assertEquals(fetchResponse.getVectorsCount(), ids.size() - 1);
181+
});
182+
183+
// Clear out the test
184+
DeleteRequest deleteAllRequest = DeleteRequest.newBuilder()
185+
.setNamespace(namespace)
186+
.setDeleteAll(true)
187+
.build();
188+
189+
blockingStub.delete(deleteAllRequest);
190+
FetchRequest fetchRequest3 = FetchRequest.newBuilder().addAllIds(ids).setNamespace(namespace).build();
191+
assertWithRetry(() -> {
192+
FetchResponse fetchResponse = blockingStub.fetch(fetchRequest3);
193+
Assertions.assertEquals(fetchResponse.getVectorsCount(), 0);
194+
});
195+
}
196+
}

‎src/integration/java/io/pinecone/integration/dataPlane/UpdateAndQueryTest.java‎ renamed to ‎src/integration/java/io/pinecone/integration/dataplane/UpdateAndQueryTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
package io.pinecone.integration.dataPlane;
1+
package io.pinecone.integration.dataplane;
22

33
import com.google.protobuf.Struct;
44
import com.google.protobuf.Value;
55
import io.grpc.StatusRuntimeException;
66
import io.pinecone.PineconeConnection;
77
import io.pinecone.helpers.RandomStringBuilder;
88
import io.pinecone.proto.*;
9+
import org.junit.After;
910
import org.junit.jupiter.api.AfterAll;
1011
import org.junit.jupiter.api.BeforeAll;
1112
import org.junit.jupiter.api.Test;

‎src/integration/java/io/pinecone/integration/dataPlane/UpsertAndDeleteTest.java‎ renamed to ‎src/integration/java/io/pinecone/integration/dataplane/UpsertAndDeleteTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package io.pinecone.integration.dataPlane;
1+
package io.pinecone.integration.dataplane;
22

33
import com.google.protobuf.Struct;
44
import com.google.protobuf.Value;

‎src/integration/java/io/pinecone/integration/dataPlane/UpsertAndDescribeIndexStatsTest.java‎ renamed to ‎src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package io.pinecone.integration.dataPlane;
1+
package io.pinecone.integration.dataplane;
22

33
import io.pinecone.*;
44
import io.pinecone.proto.*;

0 commit comments

Comments
 (0)