From 8e937d4c408d809fa67c124c7fb339e4a084c00b Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Mon, 10 Jun 2024 12:40:19 +0200 Subject: [PATCH] feat(assistant): changed default model to 4o && vision support (images) --- apps/api/src/app/chat/chat.config.ts | 2 +- .../chat/chat-audio/chat-audio.component.html | 16 ++--- .../chat/chat-audio/chat-audio.component.ts | 24 +++++-- .../chat-footer/chat-footer.component.html | 5 ++ .../chat/chat-footer/chat-footer.component.ts | 3 + .../chat-message/chat-message.component.html | 29 ++++++--- .../chat-message/chat-message.component.scss | 7 +++ .../chat-message/chat-message.component.ts | 13 ++++ .../chat-messages.component.html | 6 +- .../controls/files/files.component.ts | 7 ++- .../controls/files/files.directive.ts | 3 +- .../message-content.component.html | 25 ++++++++ .../message-content.component.scss | 37 +++++++++++ .../message-content.component.spec.ts | 23 +++++++ .../message-content.component.ts | 46 ++++++++++++++ .../message-content.helpers.ts | 37 +++++++++++ .../message-content.service.ts | 43 +++++++++++++ .../chat-cloud/chat-cloud.component.ts | 2 +- .../chat-iframe/chat-iframe.component.html | 1 + .../chat-iframe/chat-iframe.component.scss | 5 +- .../chat-iframe/chat-iframe.component.ts | 4 +- .../+chat/containers/chat/chat.component.ts | 2 +- .../app/modules/+chat/shared/chat.model.ts | 4 +- .../app/modules/+chat/shared/chat.service.ts | 62 ++++++++++++++++--- .../environments/environment.development.ts | 1 + apps/spa/src/environments/environment.ts | 1 + apps/spa/src/styles/_extends/_material.scss | 4 ++ .../src/environments/environment.prod.ts | 2 +- .../src/lib/chat/chat.model.ts | 3 +- .../src/lib/chat/chat.service.ts | 8 ++- 30 files changed, 373 insertions(+), 52 deletions(-) create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.html create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.scss create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.helpers.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.service.ts diff --git a/apps/api/src/app/chat/chat.config.ts b/apps/api/src/app/chat/chat.config.ts index ed41c1e..36ff0ef 100644 --- a/apps/api/src/app/chat/chat.config.ts +++ b/apps/api/src/app/chat/chat.config.ts @@ -6,7 +6,7 @@ export const assistantParams: AssistantCreateParams = { name: '@boldare/openai-assistant', instructions: `You are a chatbot assistant. Use the general knowledge to answer questions. Speak briefly and clearly.`, tools: [{ type: 'code_interpreter' }, { type: 'file_search' }], - model: 'gpt-4-turbo', + model: 'gpt-4o', temperature: 0.05, }; diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html index d3e56c9..fb97189 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html @@ -1,9 +1,9 @@ -@if (isAudioEnabled && message) { - - @if(!isStarted) { - play_circle - } @else { - pause_circle - } - +@if (getMessageText) { + + @if (!isStarted) { + play_circle + } @else { + pause_circle + } + } diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts index 3d1dec7..2338119 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts @@ -1,14 +1,15 @@ import { Component, Input, OnInit } from '@angular/core'; +import { MatIconModule } from '@angular/material/icon'; +import { delay } from 'rxjs'; +import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant'; +import { NgClass } from '@angular/common'; +import { getMessageText } from '../../controls/message-content/message-content.helpers'; import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; import { ChatMessage, SpeechVoice, } from '../../../modules/+chat/shared/chat.model'; import { environment } from '../../../../environments/environment'; -import { MatIconModule } from '@angular/material/icon'; -import { delay } from 'rxjs'; -import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant'; -import { NgClass } from '@angular/common'; @Component({ selector: 'ai-chat-audio', @@ -19,10 +20,17 @@ import { NgClass } from '@angular/common'; }) export class ChatAudioComponent implements OnInit { @Input() message!: ChatMessage; - isAudioEnabled = environment.isAudioEnabled; isStarted = false; audio = new Audio(); + get getMessageText(): string { + if (!environment.isAudioEnabled || !this.message) { + return ''; + } + + return getMessageText(this.message); + } + constructor(private readonly chatService: ChatClientService) {} ngOnInit(): void { @@ -42,6 +50,10 @@ export class ChatAudioComponent implements OnInit { } speech(): void { + if (!this.getMessageText) { + return; + } + this.isStarted = true; if (this.audio.src) { @@ -50,7 +62,7 @@ export class ChatAudioComponent implements OnInit { } const payload: PostSpeechDto = { - content: this.message.content, + content: getMessageText(this.message), voice: SpeechVoice.Onyx, }; diff --git a/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html b/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html index d1c6fe1..1bba187 100644 --- a/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html +++ b/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html @@ -11,6 +11,11 @@ [matTooltip]="!isDisabled ? 'Add files' : ''" [isDisabled]="isDisabled" /> } + @if (isImageContentEnabled) { + + } -} +@if (message) { + @if (message.role === 'assistant') { + + } -
- +
+ @if (messageText) { + + } - @if (message.role !== chatRole.System) { - - } -
+ @if (message.role !== chatRole.System) { + + } + + @if (messageImage.length) { +
+ @for (image of messageImage; track messageImage) { +
File ID: {{ image.image_file.file_id }}
+ } +
+ } +
} diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss index 14478b0..0be069c 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss @@ -46,3 +46,10 @@ max-width: 80%; z-index: 1; } + +.chat-message__file { + border-top: 1px dashed rgba(0, 0, 0, 0.4); + margin-top: $size-2; + padding-top: $size-2; + font-size: 11px; +} diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts index cdb0d9c..1a25913 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts @@ -7,6 +7,11 @@ import { MarkdownComponent } from 'ngx-markdown'; import { ChatAudioComponent } from '../chat-audio/chat-audio.component'; import { NgClass } from '@angular/common'; import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component'; +import { + getMessageImage, + getMessageText, +} from '../../controls/message-content/message-content.helpers'; +import { ImageFileContentBlock } from 'openai/resources/beta/threads'; @Component({ selector: 'ai-chat-message', @@ -25,6 +30,14 @@ export class ChatMessageComponent { @Input() class = ''; chatRole = ChatRole; + get messageText(): string { + return getMessageText(this.message); + } + + get messageImage(): ImageFileContentBlock[] { + return getMessageImage(this.message); + } + @HostBinding('class') get getClasses(): string { return `${this.class} is-${this.message?.role || 'none'}`; } diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html index a27b0da..950882a 100644 --- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html +++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html @@ -1,10 +1,10 @@
@for (message of initialMessages.concat(messages); track message) { - - + + } @empty { - + }
diff --git a/apps/spa/src/app/components/controls/files/files.component.ts b/apps/spa/src/app/components/controls/files/files.component.ts index 41c696d..2166b43 100644 --- a/apps/spa/src/app/components/controls/files/files.component.ts +++ b/apps/spa/src/app/components/controls/files/files.component.ts @@ -1,4 +1,4 @@ -import { Component, Input, ViewChild } from '@angular/core'; +import { Component, ElementRef, Input, ViewChild } from '@angular/core'; import { MatIcon } from '@angular/material/icon'; import { AiFilesDirective } from './files.directive'; import { toSignal } from '@angular/core/rxjs-interop'; @@ -19,7 +19,7 @@ import { ControlIconComponent } from '../control-icon/control-icon.component'; styleUrl: './files.component.scss', }) export class FilesComponent { - @ViewChild('input') input!: HTMLInputElement; + @ViewChild('input') input!: ElementRef; @Input() isDisabled = false; files = toSignal(this.fileService.files$, { initialValue: [] }); @@ -37,7 +37,8 @@ export class FilesComponent { clear(event: Event): void { event.preventDefault(); event.stopPropagation(); - this.input.files = null; + this.input.nativeElement.files = null; + this.input.nativeElement.value = ''; this.fileService.clear(); } } diff --git a/apps/spa/src/app/components/controls/files/files.directive.ts b/apps/spa/src/app/components/controls/files/files.directive.ts index 80bb359..311cfa2 100644 --- a/apps/spa/src/app/components/controls/files/files.directive.ts +++ b/apps/spa/src/app/components/controls/files/files.directive.ts @@ -6,6 +6,7 @@ import { EventEmitter, Input, } from '@angular/core'; +import { MessageContent } from 'openai/src/resources/beta/threads/messages'; @Directive({ standalone: true, @@ -13,7 +14,7 @@ import { }) export class AiFilesDirective { @Output() drop$: EventEmitter = new EventEmitter(); - @Input() files: File[] = []; + @Input() files: Array = []; event = 'init'; @HostBinding('class') get getClasses(): string { diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.html b/apps/spa/src/app/components/controls/message-content/message-content.component.html new file mode 100644 index 0000000..84cd918 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.html @@ -0,0 +1,25 @@ + + image + + @if (imageContentList$().length) { + + + {{ imageContentList$().length }} + + close + + } + + + diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.scss b/apps/spa/src/app/components/controls/message-content/message-content.component.scss new file mode 100644 index 0000000..503412f --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.scss @@ -0,0 +1,37 @@ +.files__input { + display: none; +} + +.files__counter { + display: flex; + justify-content: center; + align-items: center; + background-color: var(--color-red-500); + border-radius: 50%; + min-width: 20px; + min-height: 20px; + text-align: center; + font-size: 10px; + color: var(--color-white); + position: absolute; + right: 2px; + top: 2px; + z-index: 1; + + &:hover { + .files__number { + display: none; + } + + .files__clear { + display: block; + } + } +} + +.files__clear { + display: none; + font-size: 12px; + height: 12px; + width: 12px; +} diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts new file mode 100644 index 0000000..1b8bd43 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { MessageContentComponent } from './message-content.component'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; + +describe('MessageContentComponent', () => { + let component: MessageContentComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [MessageContentComponent, HttpClientTestingModule], + }).compileComponents(); + + fixture = TestBed.createComponent(MessageContentComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.ts new file mode 100644 index 0000000..c1a5a67 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.ts @@ -0,0 +1,46 @@ +import { Component, ElementRef, Input, ViewChild } from '@angular/core'; +import { MatIcon } from '@angular/material/icon'; +import { toSignal } from '@angular/core/rxjs-interop'; +import { MessageContentService } from './message-content.service'; +import { ControlItemComponent } from '../control-item/control-item.component'; +import { ControlIconComponent } from '../control-icon/control-icon.component'; +import { AiFilesDirective } from '../files/files.directive'; + +@Component({ + selector: 'ai-message-content', + standalone: true, + imports: [ + MatIcon, + AiFilesDirective, + ControlItemComponent, + ControlIconComponent, + ], + templateUrl: './message-content.component.html', + styleUrl: './message-content.component.scss', +}) +export class MessageContentComponent { + @ViewChild('input') input!: ElementRef; + @Input() isDisabled = false; + imageContentList$ = toSignal(this.messageContentService.data$, { + initialValue: [], + }); + + constructor(private readonly messageContentService: MessageContentService) {} + + addFiles(files: FileList) { + this.messageContentService.add(files); + } + + onFileChange(event: Event) { + const input = event.target as HTMLInputElement; + this.addFiles(input.files as FileList); + } + + clear(event: Event): void { + event.preventDefault(); + event.stopPropagation(); + this.input.nativeElement.files = null; + this.input.nativeElement.value = ''; + this.messageContentService.clear(); + } +} diff --git a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts new file mode 100644 index 0000000..4505e22 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts @@ -0,0 +1,37 @@ +import { TextContentBlock } from 'openai/resources/beta/threads/messages'; +import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages'; +import { ChatMessage } from '../../../modules/+chat/shared/chat.model'; + +export function isTextContentBlock(item: { + type: string; +}): item is TextContentBlock { + return item.type === 'text'; +} + +export function isImageFileContentBlock(item: { + type: string; +}): item is ImageFileContentBlock { + return item.type === 'image_file'; +} + +export const getMessageText = (message: ChatMessage): string => { + if (typeof message.content === 'string') { + return message.content; + } + + // @TODO: handle all types of message content + return message.content + .filter(isTextContentBlock) + .map(block => block.text.value) + .join(' '); +}; + +export const getMessageImage = ( + message: ChatMessage, +): ImageFileContentBlock[] => { + if (typeof message.content === 'string') { + return []; + } + + return message.content.filter(isImageFileContentBlock); +}; diff --git a/apps/spa/src/app/components/controls/message-content/message-content.service.ts b/apps/spa/src/app/components/controls/message-content/message-content.service.ts new file mode 100644 index 0000000..f943545 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@angular/core'; +import { BehaviorSubject } from 'rxjs'; +import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; +import { OpenAiFile } from '@boldare/openai-assistant'; + +@Injectable({ providedIn: 'root' }) +export class MessageContentService { + data$ = new BehaviorSubject([]); + + constructor(private readonly chatClientService: ChatClientService) {} + + add(files: FileList) { + const updatedFiles = [ + ...this.data$.value, + ...Object.keys(files).map(key => files[key as unknown as number]), + ]; + this.data$.next(updatedFiles); + } + + delete(index: number): void { + const updatedFiles = this.data$.value.splice(index, 1); + this.data$.next(updatedFiles); + } + + clear(): void { + this.data$.next([]); + } + + async sendFiles(): Promise { + const files = this.data$.value; + + if (!files.length) { + return []; + } + + const uploadedFilesResponse = await this.chatClientService.uploadFiles({ + files, + }); + this.clear(); + + return uploadedFilesResponse.files || []; + } +} diff --git a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts index e25496f..971cec3 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts @@ -1,7 +1,7 @@ import { Component } from '@angular/core'; +import { AssistantIframe } from '@boldare/ai-embedded'; import { ChatService } from '../../shared/chat.service'; import { environment } from '../../../../../environments/environment'; -import { AssistantIframe } from '@boldare/ai-embedded'; @Component({ selector: 'ai-chat-home', diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html index 1ddffde..002000b 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html @@ -23,6 +23,7 @@ [isDisabled]="isResponding()" [isTranscriptionEnabled]="isTranscriptionEnabled" [isAttachmentEnabled]="isAttachmentEnabled" + [isImageContentEnabled]="isImageContentEnabled" (sendMessage$)="chatService.sendMessage($event)" (sendAudio$)="chatService.sendAudio($event)" /> } diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss index 1057801..65ec27b 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss @@ -3,12 +3,13 @@ height: 100vh; max-height: 600px; background-color: var(--color-white); - border-radius: var(--border-radius-medium); + border-radius: 0 0 var(--border-radius-medium) var(--border-radius-medium); &.chat-home__iframe .chat__content, &.chat-home__iframe ::ng-deep .messages { min-height: 280px; - height: calc(100vh - 350px); + height: calc(100vh - 300px); + max-height: 500px; overflow: auto; } } diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts index be36b2f..cf95793 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts @@ -38,14 +38,14 @@ export class ChatIframeComponent implements OnInit { isAttachmentEnabled = environment.isAttachmentEnabled; isRefreshEnabled = environment.isRefreshEnabled; isConfigEnabled = environment.isConfigEnabled; - initialMessages: ChatMessage[] = []; + isImageContentEnabled = environment.isImageContentEnabled; tips = [ 'Hello! 👋 How can you help me?', 'What’s the weather like in Warsaw?', 'What is the exchange rate for USD?', - 'Show me list of Pokémon', 'Show me the stats for Pikachu (Pokémon)?', ]; + initialMessages: ChatMessage[] = []; constructor( private readonly threadService: ThreadService, diff --git a/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts b/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts index ae0800a..c53cdae 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts @@ -5,8 +5,8 @@ import { RouterModule, RouterOutlet, } from '@angular/router'; -import { ChatIframeComponent } from '../chat-iframe/chat-iframe.component'; import { ChatIframeWrapperComponent } from '../../../../components/chat/chat-iframe-wrapper/chat-iframe-wrapper.component'; +import { ChatIframeComponent } from '../chat-iframe/chat-iframe.component'; @Component({ selector: 'ai-chat', diff --git a/apps/spa/src/app/modules/+chat/shared/chat.model.ts b/apps/spa/src/app/modules/+chat/shared/chat.model.ts index 5029725..6ed369f 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.model.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.model.ts @@ -1,3 +1,5 @@ +import { MessageContent } from 'openai/resources/beta/threads'; + export interface AudioResponse { content: string; } @@ -10,7 +12,7 @@ export enum ChatRole { export interface ChatMessage { metadata?: Record; - content: string; + content: string | Array; role: ChatRole; } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts index f18306e..67a3d5a 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts @@ -9,17 +9,20 @@ import { take, tap, } from 'rxjs'; +import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model'; import { ChatGatewayService } from './chat-gateway.service'; import { ChatClientService } from './chat-client.service'; import { ThreadService } from './thread.service'; import { ChatFilesService } from './chat-files.service'; import { environment } from '../../../../environments/environment'; -import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; import { + ImageFileContentBlock, Message, - TextContentBlock, + MessageContent, + Text, } from 'openai/resources/beta/threads/messages'; +import { MessageContentService } from '../../../components/controls/message-content/message-content.service'; @Injectable({ providedIn: 'root' }) export class ChatService { @@ -34,6 +37,7 @@ export class ChatService { private readonly chatClientService: ChatClientService, private readonly threadService: ThreadService, private readonly chatFilesService: ChatFilesService, + private readonly messageContentService: MessageContentService, ) { document.body.classList.add('ai-chat'); @@ -61,7 +65,7 @@ export class ChatService { return message.content?.[0]?.type === 'text'; } - parseMessages(thread: GetThreadResponseDto): ChatMessage[] { + parseMessages(thread: GetThreadResponseDto): Message[] { if (!thread.messages) { return []; } @@ -71,11 +75,7 @@ export class ChatService { .filter( message => this.isTextMessage(message) && !this.isMessageInvisible(message), - ) - .map(message => ({ - content: (message.content[0] as TextContentBlock).text.value, - role: message.role as ChatRole, - })); + ); } setInitialValues(): void { @@ -85,7 +85,10 @@ export class ChatService { filter(threadId => !!threadId), tap(() => this.isLoading$.next(true)), mergeMap(threadId => this.threadService.getThread(threadId)), - map((response: GetThreadResponseDto) => this.parseMessages(response)), + map( + (response: GetThreadResponseDto) => + this.parseMessages(response) as ChatMessage[], + ), ) .subscribe(data => { this.messages$.next(data); @@ -142,7 +145,7 @@ export class ChatService { this.addFileMessage(files); this.chatGatewayService.callStart({ - content, + content: await this.getMessageContent(content), threadId: this.threadService.threadId$.value, attachments: files.map( file => @@ -154,6 +157,45 @@ export class ChatService { }); } + async getMessageContent(content: string): Promise { + const images = (await this.messageContentService.sendFiles()) || []; + const imageFileContentList = + images?.map( + file => + ({ + type: 'image_file', + image_file: { + file_id: file.id, + }, + }) as ImageFileContentBlock, + ) || []; + + this.messages$.next([ + ...this.messages$.value.slice(0, -1), + { + content: [ + { + type: 'text', + text: { + value: content, + annotations: [], + }, + }, + ...imageFileContentList, + ], + role: ChatRole.User, + }, + ]); + + return [ + { + type: 'text', + text: content as unknown as Text, + }, + ...imageFileContentList, + ]; + } + watchTextCreated(): Subscription { return this.chatGatewayService.textCreated().subscribe(data => { this.isTyping$.next(false); diff --git a/apps/spa/src/environments/environment.development.ts b/apps/spa/src/environments/environment.development.ts index 408b7b8..80faaea 100644 --- a/apps/spa/src/environments/environment.development.ts +++ b/apps/spa/src/environments/environment.development.ts @@ -13,4 +13,5 @@ export const environment = { isConfigEnabled: false, isAutoOpen: true, isStreamingEnabled: true, + isImageContentEnabled: true, }; diff --git a/apps/spa/src/environments/environment.ts b/apps/spa/src/environments/environment.ts index 301110c..00948a5 100644 --- a/apps/spa/src/environments/environment.ts +++ b/apps/spa/src/environments/environment.ts @@ -13,4 +13,5 @@ export const environment = { isConfigEnabled: false, isAutoOpen: true, isStreamingEnabled: true, + isImageContentEnabled: true, }; diff --git a/apps/spa/src/styles/_extends/_material.scss b/apps/spa/src/styles/_extends/_material.scss index 8b3215f..bd5de81 100644 --- a/apps/spa/src/styles/_extends/_material.scss +++ b/apps/spa/src/styles/_extends/_material.scss @@ -22,6 +22,10 @@ ); } + .mat-mdc-form-field-infix { + width: 100%; + } + mat-form-field { width: 100%; font-size: 14px; diff --git a/libs/ai-embedded/src/environments/environment.prod.ts b/libs/ai-embedded/src/environments/environment.prod.ts index 0cfe45b..7efd79e 100644 --- a/libs/ai-embedded/src/environments/environment.prod.ts +++ b/libs/ai-embedded/src/environments/environment.prod.ts @@ -1,4 +1,4 @@ export const environment = { env: 'prod', - appUrl: 'https://assistant.ai.boldare.dev', + appUrl: '', }; diff --git a/libs/openai-assistant/src/lib/chat/chat.model.ts b/libs/openai-assistant/src/lib/chat/chat.model.ts index b028f5b..d38c1cc 100644 --- a/libs/openai-assistant/src/lib/chat/chat.model.ts +++ b/libs/openai-assistant/src/lib/chat/chat.model.ts @@ -2,6 +2,7 @@ import { ApiProperty } from '@nestjs/swagger'; import { ImageFile, Message, + MessageContent, MessageCreateParams, MessageDelta, Text, @@ -58,7 +59,7 @@ export class ChatCallDto { threadId!: string; @ApiProperty() - content!: string; + content!: string | Array; @ApiProperty({ required: false }) assistantId?: string; diff --git a/libs/openai-assistant/src/lib/chat/chat.service.ts b/libs/openai-assistant/src/lib/chat/chat.service.ts index f82a3a9..36e76bb 100644 --- a/libs/openai-assistant/src/lib/chat/chat.service.ts +++ b/libs/openai-assistant/src/lib/chat/chat.service.ts @@ -7,7 +7,11 @@ import { ChatCallResponseDto, } from './chat.model'; import { ChatHelpers } from './chat.helpers'; -import { Message, MessageCreateParams } from 'openai/resources/beta/threads'; +import { + Message, + MessageContentPartParam, + MessageCreateParams, +} from 'openai/resources/beta/threads'; import { AssistantStream } from 'openai/lib/AssistantStream'; import { assistantStreamEventHandler } from '../stream/stream.utils'; @@ -44,7 +48,7 @@ export class ChatService { const { threadId, content, attachments, metadata } = payload; const message: MessageCreateParams = { role: 'user', - content, + content: content as Array, attachments, metadata, };