- @if (messageText) {
-
+ @for (msg of message.content; track $index) {
+ @if (msg.type === 'text') {
+
+ }
+
}
- @if (message.role !== chatRole.System) {
-
- }
-
- @if (messageImage.length) {
+ @if ((message | messageImageFile).length) {
- @for (image of messageImage; track messageImage) {
+ @for (
+ image of message | messageImageFile;
+ track image.image_file.file_id
+ ) {
File ID: {{ image.image_file.file_id }}
}
diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss
index 0be069c..16f476e 100644
--- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss
+++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss
@@ -11,7 +11,6 @@
justify-content: flex-end;
.chat-message {
- border-bottom-left-radius: 0;
background: var(--color-primary-200);
border-bottom-right-radius: 0;
align-self: flex-end;
diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts
index d4bf345..739e635 100644
--- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts
+++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts
@@ -1,7 +1,7 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { MarkdownModule } from 'ngx-markdown';
import { ChatMessageComponent } from './chat-message.component';
-import { MarkdownModule } from 'ngx-markdown';
describe('ChatMessageComponent', () => {
let component: ChatMessageComponent;
diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts
index 1a25913..c8af12b 100644
--- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts
+++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts
@@ -7,11 +7,9 @@ import { MarkdownComponent } from 'ngx-markdown';
import { ChatAudioComponent } from '../chat-audio/chat-audio.component';
import { NgClass } from '@angular/common';
import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component';
-import {
- getMessageImage,
- getMessageText,
-} from '../../controls/message-content/message-content.helpers';
-import { ImageFileContentBlock } from 'openai/resources/beta/threads';
+import { MessageImageFilePipe } from '../../../pipes/message-file.pipe';
+import { AnnotationPipe } from '../../../pipes/annotation.pipe';
+import { ChatAnnotationsComponent } from '../chat-annotations/chat-annotations.component';
@Component({
selector: 'ai-chat-message',
@@ -23,21 +21,16 @@ import { ImageFileContentBlock } from 'openai/resources/beta/threads';
MarkdownComponent,
ChatAudioComponent,
ChatAvatarComponent,
+ MessageImageFilePipe,
+ AnnotationPipe,
+ ChatAnnotationsComponent,
],
})
export class ChatMessageComponent {
- @Input() message!: ChatMessage;
+ @Input() message!: Partial
;
@Input() class = '';
chatRole = ChatRole;
- get messageText(): string {
- return getMessageText(this.message);
- }
-
- get messageImage(): ImageFileContentBlock[] {
- return getMessageImage(this.message);
- }
-
@HostBinding('class') get getClasses(): string {
return `${this.class} is-${this.message?.role || 'none'}`;
}
diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html
index 950882a..b97dd08 100644
--- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html
+++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html
@@ -1,5 +1,5 @@
- @for (message of initialMessages.concat(messages); track message) {
+ @for (message of initialMessages.concat(messages); track message.id) {
diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts
index 0345aff..8096c3a 100644
--- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts
+++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts
@@ -29,8 +29,8 @@ import { ChatTipsComponent } from '../chat-tips/chat-tips.component';
],
})
export class ChatMessagesComponent implements AfterViewInit, OnChanges {
- @Input() initialMessages: ChatMessage[] = [];
- @Input() messages: ChatMessage[] = [];
+ @Input() initialMessages: Partial
[] = [];
+ @Input() messages: Partial[] = [];
@Input() isTyping = false;
@Input() tips: string[] = [];
@Output() tipSelected$ = new EventEmitter();
diff --git a/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html b/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html
index da32b73..c156832 100644
--- a/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html
+++ b/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html
@@ -1,5 +1,5 @@
-@for (tip of tips; track tip) {
-
- {{ tip }}
-
+@for (tip of tips; track $index) {
+
+ {{ tip }}
+
}
diff --git a/apps/spa/src/app/components/controls/files/files.service.ts b/apps/spa/src/app/components/controls/files/files.service.ts
index e6a3552..6688115 100644
--- a/apps/spa/src/app/components/controls/files/files.service.ts
+++ b/apps/spa/src/app/components/controls/files/files.service.ts
@@ -6,9 +6,12 @@ export class FilesService {
files$ = new BehaviorSubject([]);
add(files: FileList) {
+ const convertedFiles = Object.keys(files).map(
+ key => files[key as unknown as number],
+ );
const updatedFiles = [
...this.files$.value,
- ...Object.keys(files).map(key => files[key as unknown as number]),
+ ...convertedFiles,
];
this.files$.next(updatedFiles);
diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.ts
index c1a5a67..8d068e1 100644
--- a/apps/spa/src/app/components/controls/message-content/message-content.component.ts
+++ b/apps/spa/src/app/components/controls/message-content/message-content.component.ts
@@ -1,10 +1,10 @@
import { Component, ElementRef, Input, ViewChild } from '@angular/core';
import { MatIcon } from '@angular/material/icon';
import { toSignal } from '@angular/core/rxjs-interop';
-import { MessageContentService } from './message-content.service';
import { ControlItemComponent } from '../control-item/control-item.component';
import { ControlIconComponent } from '../control-icon/control-icon.component';
import { AiFilesDirective } from '../files/files.directive';
+import { MessageContentService } from './message-content.service';
@Component({
selector: 'ai-message-content',
diff --git a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts
index 4505e22..a559979 100644
--- a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts
+++ b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts
@@ -1,37 +1,14 @@
import { TextContentBlock } from 'openai/resources/beta/threads/messages';
import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages';
-import { ChatMessage } from '../../../modules/+chat/shared/chat.model';
-export function isTextContentBlock(item: {
- type: string;
+export function isTextContentBlock(item?: {
+ type?: string;
}): item is TextContentBlock {
- return item.type === 'text';
+ return item?.type === 'text';
}
-export function isImageFileContentBlock(item: {
- type: string;
+export function isImageFileContentBlock(item?: {
+ type?: string;
}): item is ImageFileContentBlock {
- return item.type === 'image_file';
+ return item?.type === 'image_file';
}
-
-export const getMessageText = (message: ChatMessage): string => {
- if (typeof message.content === 'string') {
- return message.content;
- }
-
- // @TODO: handle all types of message content
- return message.content
- .filter(isTextContentBlock)
- .map(block => block.text.value)
- .join(' ');
-};
-
-export const getMessageImage = (
- message: ChatMessage,
-): ImageFileContentBlock[] => {
- if (typeof message.content === 'string') {
- return [];
- }
-
- return message.content.filter(isImageFileContentBlock);
-};
diff --git a/apps/spa/src/app/components/controls/message-content/message-content.service.ts b/apps/spa/src/app/components/controls/message-content/message-content.service.ts
index f943545..9d5a8f6 100644
--- a/apps/spa/src/app/components/controls/message-content/message-content.service.ts
+++ b/apps/spa/src/app/components/controls/message-content/message-content.service.ts
@@ -10,10 +10,10 @@ export class MessageContentService {
constructor(private readonly chatClientService: ChatClientService) {}
add(files: FileList) {
- const updatedFiles = [
- ...this.data$.value,
- ...Object.keys(files).map(key => files[key as unknown as number]),
- ];
+ const convertedFiles = Object.keys(files).map(
+ key => files[key as unknown as number],
+ );
+ const updatedFiles = [...this.data$.value, ...convertedFiles];
this.data$.next(updatedFiles);
}
diff --git a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts
index 6756100..9b16a34 100644
--- a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts
+++ b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts
@@ -1,9 +1,10 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { ChatCloudComponent } from './chat-cloud.component';
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { MarkdownModule } from 'ngx-markdown';
+import { ChatCloudComponent } from './chat-cloud.component';
+import { AnnotationPipe } from '../../../../pipes/annotation.pipe';
+
describe('ChatHomeComponent', () => {
let component: ChatCloudComponent;
let fixture: ComponentFixture;
@@ -15,6 +16,7 @@ describe('ChatHomeComponent', () => {
HttpClientTestingModule,
MarkdownModule.forRoot(),
],
+ providers: [AnnotationPipe],
}).compileComponents();
fixture = TestBed.createComponent(ChatCloudComponent);
diff --git a/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts
index 1136e75..a9baf12 100644
--- a/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts
+++ b/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts
@@ -4,6 +4,7 @@ import { ChatHomeComponent } from './chat-home.component';
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { MarkdownModule } from 'ngx-markdown';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { AnnotationPipe } from '../../../../pipes/annotation.pipe';
describe('ChatHomeComponent', () => {
let component: ChatHomeComponent;
@@ -17,6 +18,7 @@ describe('ChatHomeComponent', () => {
BrowserAnimationsModule,
MarkdownModule.forRoot(),
],
+ providers: [AnnotationPipe],
}).compileComponents();
fixture = TestBed.createComponent(ChatHomeComponent);
diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts
index 2c2a199..00e19d3 100644
--- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts
+++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts
@@ -4,6 +4,7 @@ import { ChatIframeComponent } from './chat-iframe.component';
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
import { RouterTestingModule } from '@angular/router/testing';
+import { AnnotationPipe } from '../../../../pipes/annotation.pipe';
describe('ChatIframeComponent', () => {
let component: ChatIframeComponent;
@@ -17,6 +18,7 @@ describe('ChatIframeComponent', () => {
ChatIframeComponent,
RouterTestingModule,
],
+ providers: [AnnotationPipe],
}).compileComponents();
fixture = TestBed.createComponent(ChatIframeComponent);
diff --git a/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts b/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts
index 405ba4a..e7dd335 100644
--- a/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts
+++ b/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts
@@ -10,6 +10,7 @@ import {
UploadFilesPayload,
UploadFilesResponseDto,
} from '@boldare/openai-assistant';
+import { FileObject } from 'openai/resources';
@Injectable({ providedIn: 'root' })
export class ChatClientService {
@@ -49,4 +50,10 @@ export class ChatClientService {
),
);
}
+
+ retriveFile(fileId: string): Observable {
+ return this.httpClient.get(
+ `${this.apiUrl}/files/retrive/${fileId}`,
+ );
+ }
}
diff --git a/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts b/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts
index e6b3728..b9569af 100644
--- a/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts
+++ b/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts
@@ -3,6 +3,7 @@ import { ChatEvents } from './chat.model';
import io from 'socket.io-client';
import {
ChatCallDto,
+ MessageWithAnnotations,
TextCreatedPayload,
TextDeltaPayload,
TextDonePayload,
@@ -37,7 +38,7 @@ export class ChatGatewayService {
return this.watchEvent(ChatEvents.TextDelta);
}
- textDone(): Observable {
+ textDone(): Observable> {
return this.watchEvent(ChatEvents.TextDone);
}
}
diff --git a/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts b/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts
new file mode 100644
index 0000000..f56fcb5
--- /dev/null
+++ b/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts
@@ -0,0 +1,43 @@
+import {
+ ImageFileContentBlock,
+ MessageContent,
+ MessageCreateParams,
+} from 'openai/resources/beta/threads';
+import { TextContentBlock } from 'openai/src/resources/beta/threads/messages';
+import { ChatMessage, ChatRole } from './chat.model';
+import { CodeInterpreterTool, FileSearchTool } from 'openai/resources/beta';
+
+export const textContentBlock = (content: string): TextContentBlock => ({
+ type: 'text',
+ text: {
+ value: content,
+ annotations: [],
+ },
+});
+
+export const imageFileContentBlock = (
+ fileId: string,
+): ImageFileContentBlock => ({
+ type: 'image_file',
+ image_file: {
+ file_id: fileId,
+ },
+});
+
+export const messageAttachment = (
+ fileId: string,
+ tools: Array = [
+ { type: 'code_interpreter' },
+ ],
+): MessageCreateParams.Attachment => ({
+ file_id: fileId,
+ tools,
+});
+
+export const messageContentBlock = (
+ content: MessageContent[],
+ role: ChatRole,
+): Partial => ({
+ content,
+ role,
+});
diff --git a/apps/spa/src/app/modules/+chat/shared/chat.model.ts b/apps/spa/src/app/modules/+chat/shared/chat.model.ts
index 6ed369f..c6ff091 100644
--- a/apps/spa/src/app/modules/+chat/shared/chat.model.ts
+++ b/apps/spa/src/app/modules/+chat/shared/chat.model.ts
@@ -1,4 +1,5 @@
-import { MessageContent } from 'openai/resources/beta/threads';
+import { AnnotationData } from '@boldare/openai-assistant';
+import { Message } from 'openai/resources/beta/threads';
export interface AudioResponse {
content: string;
@@ -7,12 +8,10 @@ export interface AudioResponse {
export enum ChatRole {
User = 'user',
Assistant = 'assistant',
- System = 'system',
}
-export interface ChatMessage {
- metadata?: Record;
- content: string | Array;
+export interface ChatMessage extends Message {
+ annotations?: AnnotationData[];
role: ChatRole;
}
diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts
index 67a3d5a..f7bc963 100644
--- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts
+++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts
@@ -9,20 +9,27 @@ import {
take,
tap,
} from 'rxjs';
+import {
+ Message,
+ MessageContent,
+ Text,
+ TextContentBlock,
+} from 'openai/resources/beta/threads/messages';
import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant';
import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model';
import { ChatGatewayService } from './chat-gateway.service';
import { ChatClientService } from './chat-client.service';
import { ThreadService } from './thread.service';
import { ChatFilesService } from './chat-files.service';
+import { MessageContentService } from '../../../components/controls/message-content/message-content.service';
import { environment } from '../../../../environments/environment';
import {
- ImageFileContentBlock,
- Message,
- MessageContent,
- Text,
-} from 'openai/resources/beta/threads/messages';
-import { MessageContentService } from '../../../components/controls/message-content/message-content.service';
+ imageFileContentBlock,
+ messageAttachment,
+ messageContentBlock,
+ textContentBlock,
+} from './chat.helpers';
+import { isTextContentBlock } from '../../../components/controls/message-content/message-content.helpers';
@Injectable({ providedIn: 'root' })
export class ChatService {
@@ -30,7 +37,7 @@ export class ChatService {
isVisible$ = new BehaviorSubject(environment.isAutoOpen);
isTyping$ = new BehaviorSubject(false);
isResponding$ = new BehaviorSubject(false);
- messages$ = new BehaviorSubject([]);
+ messages$ = new BehaviorSubject[]>([]);
constructor(
private readonly chatGatewayService: ChatGatewayService,
@@ -61,10 +68,6 @@ export class ChatService {
return metadata?.['status'] === ChatMessageStatus.Invisible;
}
- isTextMessage(message: Message): boolean {
- return message.content?.[0]?.type === 'text';
- }
-
parseMessages(thread: GetThreadResponseDto): Message[] {
if (!thread.messages) {
return [];
@@ -72,10 +75,7 @@ export class ChatService {
return thread.messages
.reverse()
- .filter(
- message =>
- this.isTextMessage(message) && !this.isMessageInvisible(message),
- );
+ .filter(message => !this.isMessageInvisible(message));
}
setInitialValues(): void {
@@ -119,7 +119,7 @@ export class ChatService {
window?.top?.postMessage('changeView', '*');
}
- addMessage(message: ChatMessage): void {
+ addMessage(message: Partial): void {
this.messages$.next([...this.messages$.value, message]);
}
@@ -127,19 +127,14 @@ export class ChatService {
if (!files?.length) {
return;
}
-
- this.addMessage({
- content: `The user has attached files to the message: ${files
- .map(file => file.filename)
- .join(', ')}`,
- role: ChatRole.System,
- });
}
async sendMessage(content: string, role = ChatRole.User): Promise {
this.isTyping$.next(true);
this.isResponding$.next(true);
- this.addMessage({ content, role });
+
+ const message = messageContentBlock([textContentBlock(content)], role);
+ this.addMessage(message);
const files = await this.chatFilesService.sendFiles();
this.addFileMessage(files);
@@ -147,44 +142,21 @@ export class ChatService {
this.chatGatewayService.callStart({
content: await this.getMessageContent(content),
threadId: this.threadService.threadId$.value,
- attachments: files.map(
- file =>
- ({
- file_id: file.id,
- tools: [{ type: 'code_interpreter' }],
- }) || [],
- ),
+ attachments: files.map(file => messageAttachment(file.id) || []),
});
}
async getMessageContent(content: string): Promise {
const images = (await this.messageContentService.sendFiles()) || [];
const imageFileContentList =
- images?.map(
- file =>
- ({
- type: 'image_file',
- image_file: {
- file_id: file.id,
- },
- }) as ImageFileContentBlock,
- ) || [];
+ images?.map(file => imageFileContentBlock(file.id)) || [];
this.messages$.next([
...this.messages$.value.slice(0, -1),
- {
- content: [
- {
- type: 'text',
- text: {
- value: content,
- annotations: [],
- },
- },
- ...imageFileContentList,
- ],
- role: ChatRole.User,
- },
+ messageContentBlock(
+ [textContentBlock(content), ...imageFileContentList],
+ ChatRole.User,
+ ),
]);
return [
@@ -200,7 +172,12 @@ export class ChatService {
return this.chatGatewayService.textCreated().subscribe(data => {
this.isTyping$.next(false);
this.isResponding$.next(true);
- this.addMessage({ content: data.text.value, role: ChatRole.Assistant });
+
+ const message = messageContentBlock(
+ [textContentBlock('')],
+ ChatRole.Assistant,
+ );
+ this.addMessage(message);
});
}
@@ -208,19 +185,30 @@ export class ChatService {
return this.chatGatewayService.textDelta().subscribe(data => {
const length = this.messages$.value.length;
this.isResponding$.next(true);
- this.messages$.value[length - 1].content = data.text.value;
+
+ const lastMessageContent = this.messages$.value[length - 1]?.content?.[0];
+
+ if (isTextContentBlock(lastMessageContent)) {
+ (
+ this.messages$.value[length - 1].content?.[0] as TextContentBlock
+ ).text.value += data.textDelta.value || '';
+ (
+ this.messages$.value[length - 1].content?.[0] as TextContentBlock
+ ).text.annotations = data.text.annotations || [];
+ }
});
}
watchTextDone(): Subscription {
- return this.chatGatewayService.textDone().subscribe(data => {
+ return this.chatGatewayService.textDone().subscribe(event => {
this.isTyping$.next(false);
this.isResponding$.next(false);
+
this.messages$.next([
...this.messages$.value.slice(0, -1),
{
- content: data.text.value,
- role: ChatRole.Assistant,
+ ...this.messages$.value.pop(),
+ annotations: event.annotations,
},
]);
});
diff --git a/apps/spa/src/app/pipes/annotation.pipe.ts b/apps/spa/src/app/pipes/annotation.pipe.ts
new file mode 100644
index 0000000..363e7ca
--- /dev/null
+++ b/apps/spa/src/app/pipes/annotation.pipe.ts
@@ -0,0 +1,64 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import {
+ FileCitationAnnotation,
+ FilePathAnnotation,
+ MessageContent,
+} from 'openai/resources/beta/threads';
+import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers';
+
+export const isFileCitation = (item: {
+ type: string;
+}): item is FileCitationAnnotation => item.type === 'file_citation';
+
+export const isFilePath = (item: {
+ type: string;
+}): item is FilePathAnnotation => item.type === 'file_path';
+
+@Pipe({
+ standalone: true,
+ name: 'annotation',
+ pure: false,
+})
+export class AnnotationPipe implements PipeTransform {
+ transform(textContent: MessageContent): string {
+ if (!isTextContentBlock(textContent)) {
+ return '';
+ }
+
+ if (!textContent.text.annotations?.length) {
+ return textContent.text.value;
+ }
+
+ let index = 1;
+
+ for (const annotation of textContent.text.annotations) {
+ const { text } = annotation;
+ let fileId = null;
+
+ if (isFileCitation(annotation)) {
+ fileId = annotation.file_citation.file_id;
+ }
+
+ if (isFilePath(annotation)) {
+ fileId = annotation.file_path.file_id;
+ }
+
+ const annotationBlock = `
+
+ ${fileId}
+ ${annotation?.type}
+
+ [${index}]
+ `;
+
+ textContent.text.value = textContent.text.value.replace(
+ text,
+ annotationBlock,
+ );
+
+ index++;
+ }
+
+ return textContent.text.value;
+ }
+}
diff --git a/apps/spa/src/app/pipes/message-file.pipe.ts b/apps/spa/src/app/pipes/message-file.pipe.ts
new file mode 100644
index 0000000..f420c99
--- /dev/null
+++ b/apps/spa/src/app/pipes/message-file.pipe.ts
@@ -0,0 +1,19 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import { isImageFileContentBlock } from '../components/controls/message-content/message-content.helpers';
+import { ChatMessage } from '../modules/+chat/shared/chat.model';
+import { ImageFileContentBlock } from 'openai/resources/beta/threads';
+
+@Pipe({
+ standalone: true,
+ name: 'messageImageFile',
+ pure: true,
+})
+export class MessageImageFilePipe implements PipeTransform {
+ transform(message: Partial): ImageFileContentBlock[] {
+ if (typeof message.content === 'string') {
+ return [];
+ }
+
+ return message?.content?.filter(isImageFileContentBlock) || [];
+ }
+}
diff --git a/apps/spa/src/app/pipes/message-text.pipe.ts b/apps/spa/src/app/pipes/message-text.pipe.ts
new file mode 100644
index 0000000..6dd3463
--- /dev/null
+++ b/apps/spa/src/app/pipes/message-text.pipe.ts
@@ -0,0 +1,24 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers';
+import { ChatMessage } from '../modules/+chat/shared/chat.model';
+
+@Pipe({
+ standalone: true,
+ name: 'messageText',
+ pure: false,
+})
+export class MessageTextPipe implements PipeTransform {
+ transform(message: Partial): string {
+ if (typeof message.content === 'string') {
+ return message.content;
+ }
+
+ // @TODO: handle all types of message content
+ return (
+ message.content
+ ?.filter(isTextContentBlock)
+ ?.map(block => block.text.value)
+ ?.join(' ') || ''
+ );
+ }
+}
diff --git a/apps/spa/src/styles/_extends/_markdown.scss b/apps/spa/src/styles/_extends/_markdown.scss
new file mode 100644
index 0000000..5d63ec1
--- /dev/null
+++ b/apps/spa/src/styles/_extends/_markdown.scss
@@ -0,0 +1,15 @@
+.annotation {
+ display: inline-block;
+ text-align: center;
+ padding: 2px 4px;
+ font-size: 11px;
+ font-weight: 400;
+ min-width: 14px;
+ border-radius: 4px;
+ text-decoration: none;
+ background-color: rgba(0, 0, 0, 0.1);
+}
+
+.annotation__metadata {
+ display: none;
+}
diff --git a/apps/spa/src/styles/_extends/_material.scss b/apps/spa/src/styles/_extends/_material.scss
index bd5de81..b1d3105 100644
--- a/apps/spa/src/styles/_extends/_material.scss
+++ b/apps/spa/src/styles/_extends/_material.scss
@@ -84,4 +84,26 @@
.mat-mdc-text-field-wrapper .mat-mdc-form-field-flex .mat-mdc-floating-label {
top: 28px;
}
+
+ .popover {
+ background: var(--color-grey-600);
+ transition: none;
+ padding: 4px 12px;
+ margin-bottom: 4px;
+ margin-left: -20px;
+ color: white;
+
+ .mat-mdc-menu-content {
+ align-items: center;
+ display: flex;
+ padding: 0;
+ font-size: 12px;
+ }
+
+ .icon {
+ font-size: inherit;
+ height: initial;
+ width: initial;
+ }
+ }
}
diff --git a/apps/spa/src/styles/styles.scss b/apps/spa/src/styles/styles.scss
index 00717f8..c0e3c3d 100644
--- a/apps/spa/src/styles/styles.scss
+++ b/apps/spa/src/styles/styles.scss
@@ -6,6 +6,7 @@
@import '_settings/borders';
@import '_extends/material';
+@import '_extends/markdown';
html,
body {
diff --git a/libs/openai-assistant/src/index.ts b/libs/openai-assistant/src/index.ts
index c7b0f65..cd8c72c 100644
--- a/libs/openai-assistant/src/index.ts
+++ b/libs/openai-assistant/src/index.ts
@@ -5,3 +5,4 @@ export * from './lib/chat';
export * from './lib/run';
export * from './lib/files';
export * from './lib/threads';
+export * from './lib/annotations';
diff --git a/libs/openai-assistant/src/lib/annotations/annotations.model.ts b/libs/openai-assistant/src/lib/annotations/annotations.model.ts
new file mode 100644
index 0000000..4956421
--- /dev/null
+++ b/libs/openai-assistant/src/lib/annotations/annotations.model.ts
@@ -0,0 +1,13 @@
+import { FileObject } from 'openai/resources';
+import { Annotation } from 'openai/resources/beta/threads/messages';
+
+export interface AnnotationData {
+ annotation: Annotation;
+ index: number;
+ file: FileObject;
+}
+
+export enum AnnotationType {
+ file_citation = 'file_citation',
+ file_path = 'file_path',
+}
diff --git a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts
new file mode 100644
index 0000000..85045b9
--- /dev/null
+++ b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts
@@ -0,0 +1,51 @@
+import OpenAI from 'openai';
+import {
+ FileCitationAnnotation,
+ FilePathAnnotation,
+ Message,
+} from 'openai/resources/beta/threads';
+import { AnnotationData, AnnotationType } from './annotations.model';
+
+export const isFileCitation = (item: {
+ type: string;
+}): item is FileCitationAnnotation => item.type === 'file_citation';
+
+export const isFilePath = (item: {
+ type: string;
+}): item is FilePathAnnotation => item.type === 'file_path';
+
+export const getAnnotations = async (
+ message: Message,
+ provider: OpenAI,
+): Promise => {
+ if (message.content[0].type !== 'text') {
+ return [];
+ }
+
+ const { text } = message.content[0];
+ const { annotations } = text;
+ const annotationsData: AnnotationData[] = [];
+
+ let index = 1;
+
+ for (const annotation of annotations) {
+ let data = null;
+
+ if (isFileCitation(annotation)) {
+ data = annotation[AnnotationType.file_citation];
+ }
+
+ if (isFilePath(annotation)) {
+ data = annotation[AnnotationType.file_path];
+ }
+
+ if (data) {
+ const file = await provider.files.retrieve(data.file_id);
+ annotationsData.push({ annotation, index, file });
+ }
+
+ index++;
+ }
+
+ return annotationsData;
+};
diff --git a/libs/openai-assistant/src/lib/annotations/index.ts b/libs/openai-assistant/src/lib/annotations/index.ts
new file mode 100644
index 0000000..12056f9
--- /dev/null
+++ b/libs/openai-assistant/src/lib/annotations/index.ts
@@ -0,0 +1,2 @@
+export * from './annotations.model';
+export * from './annotations.utils';
diff --git a/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts b/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts
index 25466bc..656b443 100644
--- a/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts
@@ -28,7 +28,15 @@ describe('ChatController', () => {
jest
.spyOn(chatService, 'call')
.mockResolvedValue({} as ChatCallResponseDto);
- const payload = { content: 'Hello' } as ChatCallDto;
+ const payload = {
+ threadId: '123',
+ content: [
+ {
+ text: { value: 'Hello', annotations: [] },
+ type: 'text',
+ },
+ ],
+ } as ChatCallDto;
await chatController.call(payload);
diff --git a/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts b/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts
index ed00cb1..1785ba8 100644
--- a/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts
@@ -3,6 +3,7 @@ import { Socket } from 'socket.io';
import { ChatGateway } from './chat.gateway';
import { ChatModule } from './chat.module';
import { ChatService } from './chat.service';
+import { ChatCallDto } from './chat.model';
describe('ChatGateway', () => {
let chatGateway: ChatGateway;
@@ -17,9 +18,15 @@ describe('ChatGateway', () => {
chatService = moduleRef.get(ChatService);
chatGateway = new ChatGateway(chatService);
- jest
- .spyOn(chatService, 'call')
- .mockResolvedValue({ threadId: '123', content: 'Hello' });
+ jest.spyOn(chatService, 'call').mockResolvedValue({
+ threadId: '123',
+ content: [
+ {
+ text: { value: 'Hello', annotations: [] },
+ type: 'text',
+ },
+ ],
+ });
});
it('should be defined', () => {
@@ -28,7 +35,15 @@ describe('ChatGateway', () => {
describe('listenForMessages', () => {
it('should call chatService.call', async () => {
- const request = { threadId: '123', content: 'Hello' };
+ const request = {
+ threadId: '123',
+ content: [
+ {
+ text: { value: 'Hello', annotations: [] },
+ type: 'text',
+ },
+ ],
+ } as ChatCallDto;
await chatGateway.listenForMessages(request, {} as Socket);
diff --git a/libs/openai-assistant/src/lib/chat/chat.gateway.ts b/libs/openai-assistant/src/lib/chat/chat.gateway.ts
index cb7bc96..a27e790 100644
--- a/libs/openai-assistant/src/lib/chat/chat.gateway.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.gateway.ts
@@ -24,12 +24,14 @@ import {
RunStepCreatedPayload,
RunStepDeltaPayload,
RunStepDonePayload,
+ MessageWithAnnotations,
} from './chat.model';
import { ChatService } from './chat.service';
import {
CodeInterpreterToolCallDelta,
FunctionToolCallDelta,
} from 'openai/resources/beta/threads/runs';
+import { getAnnotations } from '../annotations/annotations.utils';
export class ChatGateway implements OnGatewayConnection {
@WebSocketServer() server!: Server;
@@ -132,7 +134,18 @@ export class ChatGateway implements OnGatewayConnection {
socketId: string,
@MessageBody() data: MessageDonePayload,
) {
- this.server.to(socketId).emit(ChatEvents.MessageDone, data);
+ const annotations = await getAnnotations(
+ data.message,
+ this.chatsService.provider,
+ );
+ const messageWithAnnotations: MessageWithAnnotations = {
+ data,
+ annotations,
+ };
+
+ this.server
+ .to(socketId)
+ .emit(ChatEvents.MessageDone, messageWithAnnotations);
this.log(
`Socket "${ChatEvents.MessageDone}" | threadId: ${data.message.thread_id}`,
);
@@ -152,7 +165,16 @@ export class ChatGateway implements OnGatewayConnection {
}
async emitTextDone(socketId: string, @MessageBody() data: TextDonePayload) {
- this.server.to(socketId).emit(ChatEvents.TextDone, data);
+ const annotations = await getAnnotations(
+ data.message,
+ this.chatsService.provider,
+ );
+ const messageWithAnnotations: MessageWithAnnotations = {
+ data,
+ annotations,
+ };
+
+ this.server.to(socketId).emit(ChatEvents.TextDone, messageWithAnnotations);
this.log(
`Socket "${ChatEvents.TextDone}" | threadId: ${data.message?.thread_id} | ${data.text?.value}`,
);
diff --git a/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts b/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts
index 6630e10..3ccaeca 100644
--- a/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts
@@ -4,6 +4,7 @@ import { PagePromise } from 'openai/core';
import { ChatModule } from './chat.module';
import { ChatHelpers } from './chat.helpers';
import { AiService } from '../ai';
+import { error } from 'console';
describe('ChatService', () => {
let chatbotHelpers: ChatHelpers;
@@ -23,7 +24,7 @@ describe('ChatService', () => {
});
describe('getAnswer', () => {
- it('should return a string', async () => {
+ it('should return array of MessageContent', async () => {
const threadMessage: Message = {
content: [
{
@@ -44,17 +45,17 @@ describe('ChatService', () => {
} as unknown as Message;
jest
- .spyOn(chatbotHelpers, 'getLastMessage')
+ .spyOn(chatbotHelpers, 'geRunMessage')
.mockReturnValue(Promise.resolve(threadMessage));
const result = await chatbotHelpers.getAnswer({} as Run);
- expect(result).toBe('Hello');
+ expect(result).toBe(threadMessage.content);
});
});
describe('parseThreadMessage', () => {
- it('should return a string', () => {
+ it('should return a array of MessageContent', () => {
const threadMessage: Message = {
content: [
{
@@ -64,36 +65,21 @@ describe('ChatService', () => {
annotations: [],
},
},
- {
- type: 'text',
- text: {
- value: 'Hello 2',
- annotations: [],
- },
- },
],
} as unknown as Message;
const result = chatbotHelpers.parseThreadMessage(threadMessage);
- expect(result).toBe('Hello');
- });
-
- it('should return a default message', () => {
- const result = chatbotHelpers.parseThreadMessage();
-
- expect(result).toBe(
- `Seems I'm lost, would you mind reformulating your question`,
- );
+ expect(result).toBe(threadMessage.content);
});
});
- describe('getLastMessage', () => {
+ describe('geRunMessage', () => {
it('should return a ThreadMessage', async () => {
const threadMessagesPage = {
data: [
{ run_id: '1', role: 'assistant', id: '1' },
- { run_id: '1', role: 'user', id: '2' },
+ { run_id: '2', role: 'user', id: '2' },
{ run_id: '1', role: 'assistant', id: '3' },
],
} as unknown as MessagesPage;
@@ -104,7 +90,7 @@ describe('ChatService', () => {
threadMessagesPage as unknown as PagePromise,
);
- const result = await chatbotHelpers.getLastMessage({ id: '1' } as Run);
+ const result = await chatbotHelpers.geRunMessage({ id: '1' } as Run);
expect(result).toBe(threadMessagesPage.data[2]);
});
@@ -123,7 +109,7 @@ describe('ChatService', () => {
threadMessagesPage as unknown as PagePromise,
);
- const result = await chatbotHelpers.getLastMessage({ id: '1' } as Run);
+ const result = await chatbotHelpers.geRunMessage({ id: '1' } as Run);
expect(result).toBe(undefined);
});
diff --git a/libs/openai-assistant/src/lib/chat/chat.helpers.ts b/libs/openai-assistant/src/lib/chat/chat.helpers.ts
index a23520e..d4e00fe 100644
--- a/libs/openai-assistant/src/lib/chat/chat.helpers.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.helpers.ts
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
-import { Message, Run, TextContentBlock } from 'openai/resources/beta/threads';
+import { Message, MessageContent, Run } from 'openai/resources/beta/threads';
import { AiService } from '../ai';
@Injectable()
@@ -9,21 +9,20 @@ export class ChatHelpers {
constructor(private readonly aiService: AiService) {}
- async getAnswer(run: Run): Promise {
- const lastThreadMessage = await this.getLastMessage(run);
+ async getAnswer(run: Run): Promise {
+ const lastThreadMessage = await this.geRunMessage(run);
return this.parseThreadMessage(lastThreadMessage);
}
- parseThreadMessage(message?: Message): string {
+ parseThreadMessage(message?: Message): MessageContent[] {
if (!message) {
- return `Seems I'm lost, would you mind reformulating your question`;
+ throw `Seems I'm lost, would you mind reformulating your question`;
}
- const content = message.content[0] as TextContentBlock;
- return content.text.value;
+ return message.content;
}
- async getLastMessage(
+ async geRunMessage(
run: Run,
role = 'assistant',
): Promise {
diff --git a/libs/openai-assistant/src/lib/chat/chat.model.ts b/libs/openai-assistant/src/lib/chat/chat.model.ts
index d38c1cc..1646932 100644
--- a/libs/openai-assistant/src/lib/chat/chat.model.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.model.ts
@@ -14,6 +14,7 @@ import {
ToolCallDelta,
} from 'openai/resources/beta/threads/runs';
import { RunStep } from 'openai/resources/beta/threads/runs/steps';
+import { AnnotationData } from '../annotations/annotations.model';
export interface ChatAudio {
file: File;
@@ -51,7 +52,7 @@ export class ChatCallResponseDto {
threadId!: string;
@ApiProperty()
- content!: string;
+ content!: Array;
}
export class ChatCallDto {
@@ -59,7 +60,7 @@ export class ChatCallDto {
threadId!: string;
@ApiProperty()
- content!: string | Array;
+ content!: Array;
@ApiProperty({ required: false })
assistantId?: string;
@@ -146,3 +147,8 @@ export interface ChatCallCallbacks {
[ChatEvents.RunStepDelta]?: (data: RunStepDeltaPayload) => Promise;
[ChatEvents.RunStepDone]?: (data: RunStepDonePayload) => Promise;
}
+
+export interface MessageWithAnnotations {
+ data: T;
+ annotations: AnnotationData[];
+}
diff --git a/libs/openai-assistant/src/lib/chat/chat.service.spec.ts b/libs/openai-assistant/src/lib/chat/chat.service.spec.ts
index a64162c..f1b9715 100644
--- a/libs/openai-assistant/src/lib/chat/chat.service.spec.ts
+++ b/libs/openai-assistant/src/lib/chat/chat.service.spec.ts
@@ -1,6 +1,6 @@
import { Test } from '@nestjs/testing';
import { APIPromise } from 'openai/core';
-import { Message, Run } from 'openai/resources/beta/threads';
+import { Message, MessageContent, Run } from 'openai/resources/beta/threads';
import { AssistantStream } from 'openai/lib/AssistantStream';
import { AiModule } from './../ai/ai.module';
import { ChatModule } from './chat.module';
@@ -29,9 +29,17 @@ describe('ChatService', () => {
jest.spyOn(runService, 'resolve').mockReturnThis();
- jest
- .spyOn(chatbotHelpers, 'getAnswer')
- .mockReturnValue(Promise.resolve('Hello response') as Promise);
+ jest.spyOn(chatbotHelpers, 'getAnswer').mockReturnValue(
+ Promise.resolve([
+ {
+ type: 'text',
+ text: {
+ value: 'Hello response',
+ annotations: [],
+ },
+ },
+ ]) as Promise,
+ );
jest
.spyOn(chatService.threads.messages, 'create')
@@ -49,7 +57,18 @@ describe('ChatService', () => {
describe('call', () => {
it('should create "thread run"', async () => {
- const payload = { content: 'Hello', threadId: '1' } as ChatCallDto;
+ const payload = {
+ content: [
+ {
+ type: 'text',
+ text: {
+ value: 'Hello',
+ annotations: [],
+ },
+ },
+ ],
+ threadId: '1',
+ } as ChatCallDto;
const spyOnThreadRunsCreate = jest
.spyOn(chatService.threads.messages, 'create')
.mockResolvedValue({} as Message);
@@ -60,14 +79,36 @@ describe('ChatService', () => {
});
it('should return ChatCallResponse', async () => {
- const payload = { content: 'Hello', threadId: '1' } as ChatCallDto;
+ const payload = {
+ content: [
+ {
+ type: 'text',
+ text: {
+ value: 'Hello response',
+ annotations: [],
+ },
+ },
+ ],
+ threadId: '1',
+ } as ChatCallDto;
jest
.spyOn(chatService.threads.runs, 'create')
.mockResolvedValue({} as Run);
const result = await chatService.call(payload);
- expect(result).toEqual({ content: 'Hello response', threadId: '1' });
+ expect(result).toEqual({
+ content: [
+ {
+ type: 'text',
+ text: {
+ value: 'Hello response',
+ annotations: [],
+ },
+ },
+ ],
+ threadId: '1',
+ });
});
});
diff --git a/libs/openai-assistant/src/lib/files/files.controller.ts b/libs/openai-assistant/src/lib/files/files.controller.ts
index b899215..0104121 100644
--- a/libs/openai-assistant/src/lib/files/files.controller.ts
+++ b/libs/openai-assistant/src/lib/files/files.controller.ts
@@ -1,5 +1,7 @@
import {
Controller,
+ Get,
+ Param,
Post,
UploadedFiles,
UseInterceptors,
@@ -8,6 +10,7 @@ import { FileFieldsInterceptor } from '@nestjs/platform-express';
import { FilesService } from './files.service';
import { UploadFilesDto, UploadFilesResponseDto } from './files.model';
import { ApiBody, ApiConsumes, ApiResponse, ApiTags } from '@nestjs/swagger';
+import { FileObject } from 'openai/resources';
@ApiTags('Files')
@Controller('assistant/files')
@@ -26,4 +29,9 @@ export class FilesController {
files: await this.filesService.files(uploadedData.files),
};
}
+
+ @Get('/retrive/:fileId')
+ async retriveFile(@Param() params: { fileId: string }): Promise {
+ return this.filesService.retriveFile(params.fileId);
+ }
}
diff --git a/libs/openai-assistant/src/lib/files/files.service.ts b/libs/openai-assistant/src/lib/files/files.service.ts
index a247df6..f9e57da 100644
--- a/libs/openai-assistant/src/lib/files/files.service.ts
+++ b/libs/openai-assistant/src/lib/files/files.service.ts
@@ -22,4 +22,8 @@ export class FilesService {
}),
);
}
+
+ async retriveFile(fileId: string): Promise {
+ return await this.provider.files.retrieve(fileId);
+ }
}
diff --git a/package.json b/package.json
index f6d134b..b6fed07 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,9 @@
"cp:readme": "cp ./README.md ./dist/libs/openai-assistant/README.md",
"lint": "nx run-many --parallel --target=lint --all",
"test:openai-assistant": "nx test openai-assistant",
+ "test:watch:openai-assistant": "nx test openai-assistant --watch",
"test:spa": "nx test spa",
+ "test:watch:spa": "nx test spa --watch",
"test": "nx run-many --parallel --target=test --all",
"format": "nx format:write",
"prepare": "husky"