Skip to content

Gemini: send original content parts back to model #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ViacheslavKlimov merged 1 commit into from
Jan 23, 2026
Merged

Gemini: send original content parts back to model #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion ...4j-google-ai-gemini/src/main/java/dev/langchain4j/model/googleai/BaseGeminiChatModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class BaseGeminiChatModel {
protected final GeminiThinkingConfig thinkingConfig;
protected final Boolean returnThinking;
protected final boolean sendThinking;
protected final boolean sendOriginalContentParts;
protected final Integer seed;
protected final Integer logprobs;
protected final Boolean responseLogprobs;
Expand All @@ -65,6 +66,7 @@ protected BaseGeminiChatModel(GoogleAiGeminiChatModelBaseBuilder<?> builder, Gem
this.thinkingConfig = builder.thinkingConfig;
this.returnThinking = builder.returnThinking;
this.sendThinking = getOrDefault(builder.sendThinking, false);
this.sendOriginalContentParts = getOrDefault(builder.sendOriginalContentParts, false);
this.seed = builder.seed;
this.responseLogprobs = getOrDefault(builder.responseLogprobs, false);
this.enableEnhancedCivicAnswers = getOrDefault(builder.enableEnhancedCivicAnswers, false);
Expand Down Expand Up @@ -120,7 +122,7 @@ protected GeminiGenerateContentRequest createGenerateContentRequest(ChatRequest

GeminiContent systemInstruction = new GeminiContent(List.of(), GeminiRole.MODEL.toString());
List<GeminiContent> geminiContentList =
fromMessageToGContent(chatRequest.messages(), systemInstruction, sendThinking);
fromMessageToGContent(chatRequest.messages(), systemInstruction, sendThinking, sendOriginalContentParts);
String cachedContent = null;
if (systemInstruction.parts().isEmpty()) {
systemInstruction = null;
Expand Down Expand Up @@ -290,6 +292,7 @@ public abstract static class GoogleAiGeminiChatModelBaseBuilder<B extends Google
protected GeminiThinkingConfig thinkingConfig;
protected Boolean returnThinking;
protected Boolean sendThinking;
protected boolean sendOriginalContentParts;
protected Integer logprobs;
protected GeminiCachingConfig cachingConfig;
protected List<ChatModelListener> listeners;
Expand Down Expand Up @@ -558,6 +561,28 @@ public B returnThinking(Boolean returnThinking) {
return builder();
}

/**
* Configures whether to send the original, raw Gemini content parts back to the model
* in the chat history, bypassing LangChain4j's standard message conversion logic.
*
* <p>Standard LangChain4j conversion of {@code AiMessage} to Gemini {@code Content} handles
* "Thought Signatures" (used by reasoning models like {@code gemini-2.5-pro})
* incorrectly.</p>
*
* <p>Setting this to {@code true} ensures that the integration sends the exact list of
* {@code Content.Part} objects as received from the API, preserving the valid thought
* signatures required for the model's reasoning context.</p>
*
* @param sendOriginalContentParts {@code true} to bypass standard conversion and send
* original content parts; {@code false} to use standard
* LangChain4j logic.
* @return the builder instance.
*/
public B sendOriginalContentParts(boolean sendOriginalContentParts) {
this.sendOriginalContentParts = sendOriginalContentParts;
return builder();
}

/**
* Controls whether to send thinking/reasoning text to the LLM in follow-up requests.
*
Expand Down
2 changes: 1 addition & 1 deletion ...emini/src/main/java/dev/langchain4j/model/googleai/GoogleAiGeminiTokenCountEstimator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public int estimateTokenCountInMessages(Iterable<ChatMessage> messages) {
List<ChatMessage> allMessages = new LinkedList<>();
messages.forEach(allMessages::add);

List<GeminiContent> geminiContentList = fromMessageToGContent(allMessages, null, false);
List<GeminiContent> geminiContentList = fromMessageToGContent(allMessages, null, false, false);
GeminiCountTokensRequest countTokensRequest = new GeminiCountTokensRequest(geminiContentList, null);

return estimateTokenCount(countTokensRequest);
Expand Down
24 changes: 19 additions & 5 deletions ...google-ai-gemini/src/main/java/dev/langchain4j/model/googleai/PartsAndContentsMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import static dev.langchain4j.model.googleai.Json.fromJson;
import static java.util.stream.Collectors.joining;

import com.fasterxml.jackson.core.type.TypeReference;
import dev.langchain4j.agent.tool.ToolExecutionRequest;
import dev.langchain4j.data.image.Image;
import dev.langchain4j.data.message.AiMessage;
Expand Down Expand Up @@ -48,9 +49,11 @@ private PartsAndContentsMapper() {}
"thinking_signature"; // do not change, will break backward compatibility!
static final String GENERATED_IMAGES_KEY =
"generated_images"; // key for storing generated images in AiMessage attributes
static final String ORIGINAL_PARTS_KEY =
"original_parts"; // key for storing original content parts as returned by Gemini API

private static final CustomMimeTypesFileTypeDetector mimeTypeDetector = new CustomMimeTypesFileTypeDetector();

// Pattern to parse data URIs: data:[<mediatype>][;base64],<data>
private static final Pattern DATA_URI_PATTERN = Pattern.compile("^data:([^;,]+)(?:;[^,]*)?,(.*)$");

Expand Down Expand Up @@ -245,6 +248,8 @@ static AiMessage fromGPartsToAiMessage(
attributes.put(GENERATED_IMAGES_KEY, generatedImages);
}

attributes.put(ORIGINAL_PARTS_KEY, parts);

return AiMessage.builder()
.text(isNullOrEmpty(text) ? null : text)
.thinking(isNullOrEmpty(thinking) ? null : thinking)
Expand All @@ -254,7 +259,9 @@ static AiMessage fromGPartsToAiMessage(
}

static List<GeminiContent> fromMessageToGContent(
List<ChatMessage> messages, GeminiContent systemInstruction, boolean sendThinking) {
List<ChatMessage> messages, GeminiContent systemInstruction,
boolean sendThinking, boolean sendOriginalContentParts
) {
return messages.stream()
.map(msg -> {
switch (msg.type()) {
Expand All @@ -280,6 +287,13 @@ static List<GeminiContent> fromMessageToGContent(
case AI:
AiMessage aiMessage = (AiMessage) msg;

if (sendOriginalContentParts) {
List<GeminiContent.GeminiPart> originalParts = Json.convertValue(aiMessage.attributes().get(ORIGINAL_PARTS_KEY), new TypeReference<>() {});
if (originalParts != null) {
return new GeminiContent(originalParts, GeminiRole.MODEL.toString());
}
}

List<GeminiContent.GeminiPart> parts = new ArrayList<>();

if (sendThinking && isNotNullOrEmpty(aiMessage.thinking())) {
Expand Down Expand Up @@ -333,21 +347,21 @@ static List<GeminiContent> fromMessageToGContent(

/**
* Parses a data URI and returns a GeminiBlob with the extracted MIME type and base64 data.
*
*
* @param uri the data URI to parse (e.g., "data:image/png;base64,iVBORw0KG...")
* @return a GeminiBlob containing the MIME type and base64 data
* @throws IllegalArgumentException if the URI is not a valid data URI
*/
private static GeminiBlob parseDataUri(URI uri) {
String urlString = uri.toString();
Matcher matcher = DATA_URI_PATTERN.matcher(urlString);

if (matcher.matches()) {
String mimeType = matcher.group(1);
String base64Data = matcher.group(2);
return new GeminiBlob(mimeType, base64Data);
}

throw new IllegalArgumentException("Invalid data URI format: " + urlString);
}

Expand Down
24 changes: 12 additions & 12 deletions ...le-ai-gemini/src/test/java/dev/langchain4j/model/googleai/PartsAndContentsMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ void fromGPartsToAiMessage_handlesNonNullParts() {
@Test
void fromMessageToGContent_systemMessageWithText() {
SystemMessage msg = new SystemMessage("system text");
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(msg), null, false);
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(msg), null, false, false);
assertThat(result).hasSize(1);
assertThat(result.get(0).role()).isEqualTo("model");
assertThat(result.get(0).parts().get(0).text()).isEqualTo("system text");
Expand All @@ -119,15 +119,15 @@ void fromMessageToGContent_systemMessageWithText() {
@Test
void fromMessageToGContent_userMessageWithTextContent() {
UserMessage msg = new UserMessage(List.of(new dev.langchain4j.data.message.TextContent("user text")));
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(msg), null, false);
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(msg), null, false, false);
assertThat(result).hasSize(1);
assertThat(result.get(0).role()).isEqualTo("user");
assertThat(result.get(0).parts().get(0).text()).isEqualTo("user text");
}

@Test
void fromMessageToGContent_emptyMessageListReturnsEmpty() {
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(), null, false);
List<GeminiContent> result = PartsAndContentsMapper.fromMessageToGContent(List.of(), null, false, false);
assertThat(result).isEmpty();
}

Expand Down Expand Up @@ -405,17 +405,17 @@ void fromContentToGPart_throwsExceptionForInvalidDataUriPdf() {
void fromContentToGPart_handlesDataUriImageWithDifferentMimeTypes() {
// Test various image MIME types
String base64Data = "R0lGODlhAQABAAAAACw=";

// GIF
ImageContent gifContent = ImageContent.from("data:image/gif;base64," + base64Data);
GeminiContent.GeminiPart gifResult = PartsAndContentsMapper.fromContentToGPart(gifContent);
assertThat(gifResult.inlineData().mimeType()).isEqualTo("image/gif");

// WebP
ImageContent webpContent = ImageContent.from("data:image/webp;base64," + base64Data);
GeminiContent.GeminiPart webpResult = PartsAndContentsMapper.fromContentToGPart(webpContent);
assertThat(webpResult.inlineData().mimeType()).isEqualTo("image/webp");

// SVG
ImageContent svgContent = ImageContent.from("data:image/svg+xml;base64," + base64Data);
GeminiContent.GeminiPart svgResult = PartsAndContentsMapper.fromContentToGPart(svgContent);
Expand All @@ -426,17 +426,17 @@ void fromContentToGPart_handlesDataUriImageWithDifferentMimeTypes() {
void fromContentToGPart_handlesDataUriAudioWithDifferentMimeTypes() {
// Test various audio MIME types
String base64Data = "QXVkaW9EYXRh";

// WAV
Audio wavAudio = Audio.builder().url("data:audio/wav;base64," + base64Data).build();
GeminiContent.GeminiPart wavResult = PartsAndContentsMapper.fromContentToGPart(new AudioContent(wavAudio));
assertThat(wavResult.inlineData().mimeType()).isEqualTo("audio/wav");

// OGG
Audio oggAudio = Audio.builder().url("data:audio/ogg;base64," + base64Data).build();
GeminiContent.GeminiPart oggResult = PartsAndContentsMapper.fromContentToGPart(new AudioContent(oggAudio));
assertThat(oggResult.inlineData().mimeType()).isEqualTo("audio/ogg");

// FLAC
Audio flacAudio = Audio.builder().url("data:audio/flac;base64," + base64Data).build();
GeminiContent.GeminiPart flacResult = PartsAndContentsMapper.fromContentToGPart(new AudioContent(flacAudio));
Expand All @@ -447,17 +447,17 @@ void fromContentToGPart_handlesDataUriAudioWithDifferentMimeTypes() {
void fromContentToGPart_handlesDataUriVideoWithDifferentMimeTypes() {
// Test various video MIME types
String base64Data = "VmlkZW9EYXRh";

// MP4
Video mp4Video = Video.builder().url("data:video/mp4;base64," + base64Data).build();
GeminiContent.GeminiPart mp4Result = PartsAndContentsMapper.fromContentToGPart(new VideoContent(mp4Video));
assertThat(mp4Result.inlineData().mimeType()).isEqualTo("video/mp4");

// WebM
Video webmVideo = Video.builder().url("data:video/webm;base64," + base64Data).build();
GeminiContent.GeminiPart webmResult = PartsAndContentsMapper.fromContentToGPart(new VideoContent(webmVideo));
assertThat(webmResult.inlineData().mimeType()).isEqualTo("video/webm");

// MPEG
Video mpegVideo = Video.builder().url("data:video/mpeg;base64," + base64Data).build();
GeminiContent.GeminiPart mpegResult = PartsAndContentsMapper.fromContentToGPart(new VideoContent(mpegVideo));
Expand Down