From ba9c38f198eb0057b425f73f604777242d3d94c2 Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Tue, 22 Apr 2025 16:36:18 -0500 Subject: [PATCH 1/7] Add Java VertexAI bidi compile tests --- .../firebase/vertexai/JavaCompileTests.java | 94 ++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java b/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java index 066e672ffb8..ad523d87a17 100644 --- a/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java +++ b/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java @@ -21,8 +21,11 @@ import com.google.firebase.concurrent.FirebaseExecutors; import com.google.firebase.vertexai.FirebaseVertexAI; import com.google.firebase.vertexai.GenerativeModel; +import com.google.firebase.vertexai.LiveGenerativeModel; import com.google.firebase.vertexai.java.ChatFutures; import com.google.firebase.vertexai.java.GenerativeModelFutures; +import com.google.firebase.vertexai.java.LiveModelFutures; +import com.google.firebase.vertexai.java.LiveSessionFutures; import com.google.firebase.vertexai.type.BlockReason; import com.google.firebase.vertexai.type.Candidate; import com.google.firebase.vertexai.type.Citation; @@ -33,24 +36,33 @@ import com.google.firebase.vertexai.type.FileDataPart; import com.google.firebase.vertexai.type.FinishReason; import com.google.firebase.vertexai.type.FunctionCallPart; +import com.google.firebase.vertexai.type.FunctionResponsePart; import com.google.firebase.vertexai.type.GenerateContentResponse; +import com.google.firebase.vertexai.type.GenerationConfig; import com.google.firebase.vertexai.type.HarmCategory; import com.google.firebase.vertexai.type.HarmProbability; import com.google.firebase.vertexai.type.HarmSeverity; import com.google.firebase.vertexai.type.ImagePart; import com.google.firebase.vertexai.type.InlineDataPart; +import com.google.firebase.vertexai.type.LiveContentResponse; +import com.google.firebase.vertexai.type.LiveGenerationConfig; +import com.google.firebase.vertexai.type.MediaData; import com.google.firebase.vertexai.type.ModalityTokenCount; import com.google.firebase.vertexai.type.Part; import com.google.firebase.vertexai.type.PromptFeedback; +import com.google.firebase.vertexai.type.ResponseModality; import com.google.firebase.vertexai.type.SafetyRating; +import com.google.firebase.vertexai.type.SpeechConfig; import com.google.firebase.vertexai.type.TextPart; import com.google.firebase.vertexai.type.UsageMetadata; +import com.google.firebase.vertexai.type.Voices; import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import kotlinx.serialization.json.JsonElement; import kotlinx.serialization.json.JsonNull; +import kotlinx.serialization.json.JsonObject; import org.junit.Assert; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; @@ -63,9 +75,31 @@ public class JavaCompileTests { public void initializeJava() throws Exception { FirebaseVertexAI vertex = FirebaseVertexAI.getInstance(); - GenerativeModel model = vertex.generativeModel("fake-model-name"); + GenerativeModel model = vertex.generativeModel("fake-model-name", getConfig()); + LiveGenerativeModel live = vertex.liveModel("fake-model-name", getLiveConfig()); GenerativeModelFutures futures = GenerativeModelFutures.from(model); + LiveModelFutures liveFutures = LiveModelFutures.from(live); testFutures(futures); + testLiveFutures(liveFutures); + } + + private GenerationConfig getConfig() { + return new GenerationConfig.Builder().build(); + // TODO b/406558430 GenerationConfig.Builder.setParts returns void + } + + private LiveGenerationConfig getLiveConfig() { + return new LiveGenerationConfig.Builder() + .setTopK(10) + .setTopP(11.0F) + .setTemperature(32.0F) + .setCandidateCount(1) + .setMaxOutputTokens(0xCAFEBABE) + .setFrequencyPenalty(1.0F) + .setPresencePenalty(2.0F) + .setResponseModality(ResponseModality.AUDIO) + .setSpeechConfig(new SpeechConfig(Voices.AOEDE)) + .build(); } private void testFutures(GenerativeModelFutures futures) throws Exception { @@ -236,4 +270,62 @@ public void validateUsageMetadata(UsageMetadata metadata) { } } } + + private void testLiveFutures(LiveModelFutures futures) throws Exception { + LiveSessionFutures session = futures.connect().get(); + session + .receive() + .subscribe( + new Subscriber() { + @Override + public void onSubscribe(Subscription s) { + s.request(Long.MAX_VALUE); + } + + @Override + public void onNext(LiveContentResponse response) { + validateLiveContentResponse(response); + } + + @Override + public void onError(Throwable t) { + // Ignore + } + + @Override + public void onComplete() { + // Also ignore + } + }); + + session.send("Fake message"); + session.send(new Content.Builder().addText("Fake message").build()); + + byte[] bytes = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + session.sendMediaStream(List.of(new MediaData(bytes, "image/jxl"))); + + FunctionResponsePart functionResponse = + new FunctionResponsePart("myFunction", new JsonObject(Map.of())); + session.sendFunctionResponse(List.of(functionResponse, functionResponse)); + + session.startAudioConversation(part -> functionResponse); + session.startAudioConversation(); + session.stopAudioConversation(); + session.stopReceiving(); + session.close(); + } + + private void validateLiveContentResponse(LiveContentResponse response) { + //int status = response.getStatus(); + //Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); + //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); + //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); + // TODO b/412743328 LiveContentResponse.Status inaccessible for Java users + Content data = response.getData(); + if (data != null) { + validateContent(data); + } + String text = response.getText(); + validateFunctionCalls(response.getFunctionCalls()); + } } From d16b70522dd276caa50c61516cb791bcded15310 Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Tue, 29 Apr 2025 16:50:23 -0500 Subject: [PATCH 2/7] Copy tests to Firebase AI --- .../google/firebase/ai/JavaCompileTests.java | 100 +++++++++++++++++- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index 5e363ed95b2..8b0243b77f9 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -14,14 +14,18 @@ * limitations under the License. */ -package java.com.google.firebase.ai; +package java.com.google.firebase.vertexai; import android.graphics.Bitmap; import com.google.common.util.concurrent.ListenableFuture; +import com.google.firebase.concurrent.FirebaseExecutors; import com.google.firebase.ai.FirebaseAI; import com.google.firebase.ai.GenerativeModel; +import com.google.firebase.ai.LiveGenerativeModel; import com.google.firebase.ai.java.ChatFutures; import com.google.firebase.ai.java.GenerativeModelFutures; +import com.google.firebase.ai.java.LiveModelFutures; +import com.google.firebase.ai.java.LiveSessionFutures; import com.google.firebase.ai.type.BlockReason; import com.google.firebase.ai.type.Candidate; import com.google.firebase.ai.type.Citation; @@ -32,25 +36,33 @@ import com.google.firebase.ai.type.FileDataPart; import com.google.firebase.ai.type.FinishReason; import com.google.firebase.ai.type.FunctionCallPart; +import com.google.firebase.ai.type.FunctionResponsePart; import com.google.firebase.ai.type.GenerateContentResponse; +import com.google.firebase.ai.type.GenerationConfig; import com.google.firebase.ai.type.HarmCategory; import com.google.firebase.ai.type.HarmProbability; import com.google.firebase.ai.type.HarmSeverity; import com.google.firebase.ai.type.ImagePart; import com.google.firebase.ai.type.InlineDataPart; +import com.google.firebase.ai.type.LiveContentResponse; +import com.google.firebase.ai.type.LiveGenerationConfig; +import com.google.firebase.ai.type.MediaData; import com.google.firebase.ai.type.ModalityTokenCount; import com.google.firebase.ai.type.Part; import com.google.firebase.ai.type.PromptFeedback; +import com.google.firebase.ai.type.ResponseModality; import com.google.firebase.ai.type.SafetyRating; +import com.google.firebase.ai.type.SpeechConfig; import com.google.firebase.ai.type.TextPart; import com.google.firebase.ai.type.UsageMetadata; -import com.google.firebase.concurrent.FirebaseExecutors; +import com.google.firebase.ai.type.Voices; import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import kotlinx.serialization.json.JsonElement; import kotlinx.serialization.json.JsonNull; +import kotlinx.serialization.json.JsonObject; import org.junit.Assert; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; @@ -63,9 +75,31 @@ public class JavaCompileTests { public void initializeJava() throws Exception { FirebaseAI vertex = FirebaseAI.getInstance(); - GenerativeModel model = vertex.generativeModel("fake-model-name"); + GenerativeModel model = vertex.generativeModel("fake-model-name", getConfig()); + LiveGenerativeModel live = vertex.liveModel("fake-model-name", getLiveConfig()); GenerativeModelFutures futures = GenerativeModelFutures.from(model); + LiveModelFutures liveFutures = LiveModelFutures.from(live); testFutures(futures); + testLiveFutures(liveFutures); + } + + private GenerationConfig getConfig() { + return new GenerationConfig.Builder().build(); + // TODO b/406558430 GenerationConfig.Builder.setParts returns void + } + + private LiveGenerationConfig getLiveConfig() { + return new LiveGenerationConfig.Builder() + .setTopK(10) + .setTopP(11.0F) + .setTemperature(32.0F) + .setCandidateCount(1) + .setMaxOutputTokens(0xCAFEBABE) + .setFrequencyPenalty(1.0F) + .setPresencePenalty(2.0F) + .setResponseModality(ResponseModality.AUDIO) + .setSpeechConfig(new SpeechConfig(Voices.AOEDE)) + .build(); } private void testFutures(GenerativeModelFutures futures) throws Exception { @@ -138,7 +172,7 @@ public void validateGenerateContentResponse(GenerateContentResponse response) { List candidates = response.getCandidates(); if (candidates.size() == 1 && candidates.get(0).getContent().getParts().stream() - .anyMatch(p -> p instanceof TextPart && !((TextPart) p).getText().isEmpty())) { + .anyMatch(p -> p instanceof TextPart && !((TextPart) p).getText().isEmpty())) { String text = response.getText(); Assert.assertNotNull(text); Assert.assertFalse(text.isBlank()); @@ -236,4 +270,62 @@ public void validateUsageMetadata(UsageMetadata metadata) { } } } + + private void testLiveFutures(LiveModelFutures futures) throws Exception { + LiveSessionFutures session = futures.connect().get(); + session + .receive() + .subscribe( + new Subscriber() { + @Override + public void onSubscribe(Subscription s) { + s.request(Long.MAX_VALUE); + } + + @Override + public void onNext(LiveContentResponse response) { + validateLiveContentResponse(response); + } + + @Override + public void onError(Throwable t) { + // Ignore + } + + @Override + public void onComplete() { + // Also ignore + } + }); + + session.send("Fake message"); + session.send(new Content.Builder().addText("Fake message").build()); + + byte[] bytes = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + session.sendMediaStream(List.of(new MediaData(bytes, "image/jxl"))); + + FunctionResponsePart functionResponse = + new FunctionResponsePart("myFunction", new JsonObject(Map.of())); + session.sendFunctionResponse(List.of(functionResponse, functionResponse)); + + session.startAudioConversation(part -> functionResponse); + session.startAudioConversation(); + session.stopAudioConversation(); + session.stopReceiving(); + session.close(); + } + + private void validateLiveContentResponse(LiveContentResponse response) { + //int status = response.getStatus(); + //Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); + //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); + //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); + // TODO b/412743328 LiveContentResponse.Status inaccessible for Java users + Content data = response.getData(); + if (data != null) { + validateContent(data); + } + String text = response.getText(); + validateFunctionCalls(response.getFunctionCalls()); + } } From 88c5a8ac0f9a31b5084baa08a27f1c4072189ff3 Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Wed, 30 Apr 2025 11:06:45 -0500 Subject: [PATCH 3/7] Format --- .../com/google/firebase/ai/JavaCompileTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index 8b0243b77f9..a3236b80f87 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -18,7 +18,6 @@ import android.graphics.Bitmap; import com.google.common.util.concurrent.ListenableFuture; -import com.google.firebase.concurrent.FirebaseExecutors; import com.google.firebase.ai.FirebaseAI; import com.google.firebase.ai.GenerativeModel; import com.google.firebase.ai.LiveGenerativeModel; @@ -56,6 +55,7 @@ import com.google.firebase.ai.type.TextPart; import com.google.firebase.ai.type.UsageMetadata; import com.google.firebase.ai.type.Voices; +import com.google.firebase.concurrent.FirebaseExecutors; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -172,7 +172,7 @@ public void validateGenerateContentResponse(GenerateContentResponse response) { List candidates = response.getCandidates(); if (candidates.size() == 1 && candidates.get(0).getContent().getParts().stream() - .anyMatch(p -> p instanceof TextPart && !((TextPart) p).getText().isEmpty())) { + .anyMatch(p -> p instanceof TextPart && !((TextPart) p).getText().isEmpty())) { String text = response.getText(); Assert.assertNotNull(text); Assert.assertFalse(text.isBlank()); @@ -316,10 +316,10 @@ public void onComplete() { } private void validateLiveContentResponse(LiveContentResponse response) { - //int status = response.getStatus(); - //Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); - //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); - //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); + // int status = response.getStatus(); + // Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); + // Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); + // Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); // TODO b/412743328 LiveContentResponse.Status inaccessible for Java users Content data = response.getData(); if (data != null) { From edacab9de733da382c1ff376f55fcaedee7b454a Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Wed, 30 Apr 2025 11:43:24 -0500 Subject: [PATCH 4/7] Format --- .../com/google/firebase/vertexai/JavaCompileTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java b/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java index ad523d87a17..cf71db18798 100644 --- a/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java +++ b/firebase-vertexai/src/testUtil/java/com/google/firebase/vertexai/JavaCompileTests.java @@ -316,10 +316,10 @@ public void onComplete() { } private void validateLiveContentResponse(LiveContentResponse response) { - //int status = response.getStatus(); - //Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); - //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); - //Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); + // int status = response.getStatus(); + // Assert.assertEquals(status, LiveContentResponse.Status.Companion.getNORMAL()); + // Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getINTERRUPTED()); + // Assert.assertNotEquals(status, LiveContentResponse.Status.Companion.getTURN_COMPLETE()); // TODO b/412743328 LiveContentResponse.Status inaccessible for Java users Content data = response.getData(); if (data != null) { From 39a9bc8e04624adb845db6106d4e29c1601aaf19 Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Wed, 30 Apr 2025 13:10:36 -0500 Subject: [PATCH 5/7] Package fix --- .../testUtil/java/com/google/firebase/ai/JavaCompileTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index a3236b80f87..ad8dd9b5663 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package java.com.google.firebase.vertexai; +package java.com.google.firebase.ai; import android.graphics.Bitmap; import com.google.common.util.concurrent.ListenableFuture; From 934246ac18f288f56b01e815c2d59514824e56e0 Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Wed, 30 Apr 2025 14:31:24 -0500 Subject: [PATCH 6/7] Opt in --- .../java/com/google/firebase/ai/JavaCompileTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index ad8dd9b5663..c6d09411f60 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -49,6 +49,7 @@ import com.google.firebase.ai.type.ModalityTokenCount; import com.google.firebase.ai.type.Part; import com.google.firebase.ai.type.PromptFeedback; +import com.google.firebase.ai.type.PublicPreviewAPI; import com.google.firebase.ai.type.ResponseModality; import com.google.firebase.ai.type.SafetyRating; import com.google.firebase.ai.type.SpeechConfig; @@ -60,6 +61,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Executor; + +import kotlin.OptIn; import kotlinx.serialization.json.JsonElement; import kotlinx.serialization.json.JsonNull; import kotlinx.serialization.json.JsonObject; @@ -71,6 +74,7 @@ /** * Tests in this file exist to be compiled, not invoked */ +@OptIn(markerClass = PublicPreviewAPI.class) public class JavaCompileTests { public void initializeJava() throws Exception { From 4b4bf2745798f785cb488195eb919f5e31a0056f Mon Sep 17 00:00:00 2001 From: Emily Ploszaj Date: Wed, 30 Apr 2025 15:03:07 -0500 Subject: [PATCH 7/7] Format again --- .../testUtil/java/com/google/firebase/ai/JavaCompileTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index c6d09411f60..b369bcd6d07 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -61,7 +61,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Executor; - import kotlin.OptIn; import kotlinx.serialization.json.JsonElement; import kotlinx.serialization.json.JsonNull;