Add search scores, models, and deployments to "Thought process" tab, …

…surface additional properties (#1375) * update reqs * Add score, model, styling * Update tests * Use a new variable name to avoid type clash * Use correct variable name in thought process
Azure-Samples · Mar 7, 2024 · 3d86d24 · 3d86d24
1 parent e191f74
commit 3d86d24
Show file tree

Hide file tree

Showing 54 changed files with 961 additions and 328 deletions.
diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py
@@ -30,6 +30,8 @@ class Document:
     oids: Optional[List[str]]
     groups: Optional[List[str]]
     captions: List[QueryCaptionResult]
+    score: Optional[float] = None
+    reranker_score: Optional[float] = None
 
     def serialize_for_results(self) -> dict[str, Any]:
         return {
@@ -54,6 +56,8 @@ def serialize_for_results(self) -> dict[str, Any]:
                 if self.captions
                 else []
             ),
+            "score": self.score,
+            "reranker_score": self.reranker_score,
         }
 
     @classmethod
@@ -153,6 +157,8 @@ async def search(
                         oids=document.get("oids"),
                         groups=document.get("groups"),
                         captions=cast(List[QueryCaptionResult], document.get("@search.captions")),
+                        score=document.get("@search.score"),
+                        reranker_score=document.get("@search.reranker_score"),
                     )
                 )
         return documents

diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py
@@ -116,7 +116,7 @@ async def run_until_final_call(
         ]
 
         # STEP 1: Generate an optimized keyword search query based on the chat history and the last question
-        messages = self.get_messages_from_history(
+        query_messages = self.get_messages_from_history(
             system_prompt=self.query_prompt_template,
             model_id=self.chatgpt_model,
             history=history,
@@ -126,7 +126,7 @@ async def run_until_final_call(
         )
 
         chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
-            messages=messages,  # type: ignore
+            messages=query_messages,  # type: ignore
             # Azure Open AI takes the deployment name as the model name
             model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
             temperature=0.0,  # Minimize creativity for search query generation
@@ -179,16 +179,38 @@ async def run_until_final_call(
             "data_points": data_points,
             "thoughts": [
                 ThoughtStep(
-                    "Original user query",
-                    original_user_query,
+                    "Prompt to generate search query",
+                    [str(message) for message in query_messages],
+                    (
+                        {"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
+                        if self.chatgpt_deployment
+                        else {"model": self.chatgpt_model}
+                    ),
                 ),
                 ThoughtStep(
-                    "Generated search query",
+                    "Search using generated search query",
                     query_text,
-                    {"use_semantic_captions": use_semantic_captions, "has_vector": has_vector},
+                    {
+                        "use_semantic_captions": use_semantic_captions,
+                        "use_semantic_ranker": use_semantic_ranker,
+                        "top": top,
+                        "filter": filter,
+                        "has_vector": has_vector,
+                    },
+                ),
+                ThoughtStep(
+                    "Search results",
+                    [result.serialize_for_results() for result in results],
+                ),
+                ThoughtStep(
+                    "Prompt to generate answer",
+                    [str(message) for message in messages],
+                    (
+                        {"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
+                        if self.chatgpt_deployment
+                        else {"model": self.chatgpt_model}
+                    ),
                 ),
-                ThoughtStep("Results", [result.serialize_for_results() for result in results]),
-                ThoughtStep("Prompt", [str(message) for message in messages]),
             ],
         }
 

diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py
@@ -98,7 +98,7 @@ async def run_until_final_call(
         # STEP 1: Generate an optimized keyword search query based on the chat history and the last question
         user_query_request = "Generate search query for: " + original_user_query
 
-        messages = self.get_messages_from_history(
+        query_messages = self.get_messages_from_history(
             system_prompt=self.query_prompt_template,
             model_id=self.gpt4v_model,
             history=history,
@@ -109,7 +109,7 @@ async def run_until_final_call(
 
         chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
             model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
-            messages=messages,
+            messages=query_messages,
             temperature=0.0,  # Minimize creativity for search query generation
             max_tokens=100,
             n=1,
@@ -178,16 +178,38 @@ async def run_until_final_call(
             "data_points": data_points,
             "thoughts": [
                 ThoughtStep(
-                    "Original user query",
-                    original_user_query,
+                    "Prompt to generate search query",
+                    [str(message) for message in query_messages],
+                    (
+                        {"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
+                        if self.gpt4v_deployment
+                        else {"model": self.gpt4v_model}
+                    ),
                 ),
                 ThoughtStep(
-                    "Generated search query",
+                    "Search using generated search query",
                     query_text,
-                    {"use_semantic_captions": use_semantic_captions, "vector_fields": vector_fields},
+                    {
+                        "use_semantic_captions": use_semantic_captions,
+                        "use_semantic_ranker": use_semantic_ranker,
+                        "top": top,
+                        "filter": filter,
+                        "vector_fields": vector_fields,
+                    },
+                ),
+                ThoughtStep(
+                    "Search results",
+                    [result.serialize_for_results() for result in results],
+                ),
+                ThoughtStep(
+                    "Prompt to generate answer",
+                    [str(message) for message in messages],
+                    (
+                        {"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
+                        if self.gpt4v_deployment
+                        else {"model": self.gpt4v_model}
+                    ),
                 ),
-                ThoughtStep("Results", [result.serialize_for_results() for result in results]),
-                ThoughtStep("Prompt", [str(message) for message in messages]),
             ],
         }
 

diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py
@@ -112,12 +112,12 @@ async def run(
         message_builder.insert_message("user", user_content)
         message_builder.insert_message("assistant", self.answer)
         message_builder.insert_message("user", self.question)
-
+        updated_messages = message_builder.messages
         chat_completion = (
             await self.openai_client.chat.completions.create(
                 # Azure Open AI takes the deployment name as the model name
                 model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
-                messages=message_builder.messages,
+                messages=updated_messages,
                 temperature=overrides.get("temperature", 0.3),
                 max_tokens=1024,
                 n=1,
@@ -129,14 +129,29 @@ async def run(
             "data_points": data_points,
             "thoughts": [
                 ThoughtStep(
-                    "Search Query",
+                    "Search using user query",
                     query_text,
                     {
                         "use_semantic_captions": use_semantic_captions,
+                        "use_semantic_ranker": use_semantic_ranker,
+                        "top": top,
+                        "filter": filter,
+                        "has_vector": has_vector,
                     },
                 ),
-                ThoughtStep("Results", [result.serialize_for_results() for result in results]),
-                ThoughtStep("Prompt", [str(message) for message in message_builder.messages]),
+                ThoughtStep(
+                    "Search results",
+                    [result.serialize_for_results() for result in results],
+                ),
+                ThoughtStep(
+                    "Prompt to generate answer",
+                    [str(message) for message in updated_messages],
+                    (
+                        {"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
+                        if self.chatgpt_deployment
+                        else {"model": self.chatgpt_model}
+                    ),
+                ),
             ],
         }
 

diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py
@@ -132,11 +132,11 @@ async def run(
 
         # Append user message
         message_builder.insert_message("user", user_content)
-
+        updated_messages = message_builder.messages
         chat_completion = (
             await self.openai_client.chat.completions.create(
                 model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
-                messages=message_builder.messages,
+                messages=updated_messages,
                 temperature=overrides.get("temperature", 0.3),
                 max_tokens=1024,
                 n=1,
@@ -152,12 +152,29 @@ async def run(
             "data_points": data_points,
             "thoughts": [
                 ThoughtStep(
-                    "Search Query",
+                    "Search using user query",
                     query_text,
-                    {"use_semantic_captions": use_semantic_captions, "vector_fields": vector_fields},
+                    {
+                        "use_semantic_captions": use_semantic_captions,
+                        "use_semantic_ranker": use_semantic_ranker,
+                        "top": top,
+                        "filter": filter,
+                        "vector_fields": vector_fields,
+                    },
+                ),
+                ThoughtStep(
+                    "Search results",
+                    [result.serialize_for_results() for result in results],
+                ),
+                ThoughtStep(
+                    "Prompt to generate answer",
+                    [str(message) for message in updated_messages],
+                    (
+                        {"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
+                        if self.gpt4v_deployment
+                        else {"model": self.gpt4v_model}
+                    ),
                 ),
-                ThoughtStep("Results", [result.serialize_for_results() for result in results]),
-                ThoughtStep("Prompt", [str(message) for message in message_builder.messages]),
             ],
         }
         chat_completion["choices"][0]["context"] = extra_info

diff --git a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css
@@ -40,7 +40,8 @@
 .tStep {
     color: #123bb6;
     position: relative;
-    font-size: 12px;
+    font-size: 14px;
+    margin-bottom: 8px;
 }
 
 .tCodeBlock {
@@ -53,6 +54,7 @@
     font-size: 12px;
     padding: 3px 10px;
     border-radius: 10px;
+    margin-bottom: 8px;
 }
 
 .citationImg {

diff --git a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx
@@ -1,5 +1,4 @@
 import { Stack, Pivot, PivotItem } from "@fluentui/react";
-import SyntaxHighlighter from "react-syntax-highlighter";
 
 import styles from "./AnalysisPanel.module.css";
 

diff --git a/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx b/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx
@@ -16,22 +16,20 @@ export const ThoughtProcess = ({ thoughts }: Props) => {
                 return (
                     <li className={styles.tListItem} key={ind}>
                         <div className={styles.tStep}>{t.title}</div>
+                        <Stack horizontal tokens={{ childrenGap: 5 }}>
+                            {t.props &&
+                                (Object.keys(t.props) || []).map((k: any) => (
+                                    <span className={styles.tProp}>
+                                        {k}: {JSON.stringify(t.props?.[k])}
+                                    </span>
+                                ))}
+                        </Stack>
                         {Array.isArray(t.description) ? (
                             <SyntaxHighlighter language="json" wrapLongLines className={styles.tCodeBlock}>
                                 {JSON.stringify(t.description, null, 2)}
                             </SyntaxHighlighter>
                         ) : (
-                            <>
-                                <div>{t.description}</div>
-                                <Stack horizontal tokens={{ childrenGap: 5 }}>
-                                    {t.props &&
-                                        (Object.keys(t.props) || []).map((k: any) => (
-                                            <span className={styles.tProp}>
-                                                {k}: {JSON.stringify(t.props?.[k])}
-                                            </span>
-                                        ))}
-                                </Stack>
-                            </>
+                            <div>{t.description}</div>
                         )}
                     </li>
                 );

diff --git a/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json b/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json
@@ -11,9 +11,13 @@
                     {
                         "description": "What is the capital of France?",
                         "props": {
-                            "use_semantic_captions": false
+                            "filter": null,
+                            "has_vector": true,
+                            "top": 3,
+                            "use_semantic_captions": false,
+                            "use_semantic_ranker": null
                         },
-                        "title": "Search Query"
+                        "title": "Search using user query"
                     },
                     {
                         "description": [
@@ -32,12 +36,14 @@
                                 "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
                                 "imageEmbedding": null,
                                 "oids": null,
+                                "reranker_score": 3.4577205181121826,
+                                "score": 0.03279569745063782,
                                 "sourcefile": "Benefit_Options.pdf",
                                 "sourcepage": "Benefit_Options-2.pdf"
                             }
                         ],
                         "props": null,
-                        "title": "Results"
+                        "title": "Search results"
                     },
                     {
                         "description": [
@@ -46,8 +52,10 @@
                             "{'role': 'assistant', 'content': 'In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].'}",
                             "{'role': 'user', 'content': 'What is the capital of France?\\nSources:\\n Benefit_Options-2.pdf: There is a whistleblower policy.'}"
                         ],
-                        "props": null,
-                        "title": "Prompt"
+                        "props": {
+                            "model": "gpt-35-turbo"
+                        },
+                        "title": "Prompt to generate answer"
                     }
                 ]
             },

diff --git a/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json b/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json
@@ -11,9 +11,13 @@
                     {
                         "description": "What is the capital of France?",
                         "props": {
-                            "use_semantic_captions": false
+                            "filter": null,
+                            "has_vector": true,
+                            "top": 3,
+                            "use_semantic_captions": false,
+                            "use_semantic_ranker": null
                         },
-                        "title": "Search Query"
+                        "title": "Search using user query"
                     },
                     {
                         "description": [
@@ -32,12 +36,14 @@
                                 "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
                                 "imageEmbedding": null,
                                 "oids": null,
+                                "reranker_score": 3.4577205181121826,
+                                "score": 0.03279569745063782,
                                 "sourcefile": "Benefit_Options.pdf",
                                 "sourcepage": "Benefit_Options-2.pdf"
                             }
                         ],
                         "props": null,
-                        "title": "Results"
+                        "title": "Search results"
                     },
                     {
                         "description": [
@@ -46,8 +52,11 @@
                             "{'role': 'assistant', 'content': 'In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].'}",
                             "{'role': 'user', 'content': 'What is the capital of France?\\nSources:\\n Benefit_Options-2.pdf: There is a whistleblower policy.'}"
                         ],
-                        "props": null,
-                        "title": "Prompt"
+                        "props": {
+                            "deployment": "test-chatgpt",
+                            "model": "gpt-35-turbo"
+                        },
+                        "title": "Prompt to generate answer"
                     }
                 ]
             },