feat(dashboard): The dashboard supports displaying real-time data (#930)

Author: ziyujiahao

backend/apps/chat/api/chat.py

@@ -14,7 +14,7 @@
     list_chats, get_chat_with_records, create_chat, rename_chat, \
     delete_chat, get_chat_chart_data, get_chat_predict_data, get_chat_with_records_with_data, get_chat_record_by_id, \
     format_json_data, format_json_list_data, get_chart_config, list_recent_questions, get_chat as get_chat_exec, \
-    rename_chat_with_user, get_chat_log_history
+    rename_chat_with_user, get_chat_log_history, get_chart_data_with_user_live
 from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, ChatQuestion, AxisObj, QuickCommand, \
     ChatInfo, Chat, ChatFinishStep
 from apps.chat.task.llm import LLMService
@@ -80,6 +80,13 @@ def inner():
 
     return await asyncio.to_thread(inner)
 
+@router.get("/record/{chat_record_id}/data_live", summary=f"{PLACEHOLDER_PREFIX}get_chart_data_live")
+async def chat_record_data_live(session: SessionDep, current_user: CurrentUser, chat_record_id: int):
+    def inner():
+        data = get_chart_data_with_user_live(chat_record_id=chat_record_id, session=session, current_user=current_user)
+        return format_json_data(data)
+
+    return await asyncio.to_thread(inner)
 
 @router.get("/record/{chat_record_id}/predict_data", summary=f"{PLACEHOLDER_PREFIX}get_chart_predict_data")
 async def chat_predict_data(session: SessionDep, current_user: CurrentUser, chat_record_id: int):

backend/apps/chat/curd/chat.py

@@ -9,12 +9,15 @@
 
 from apps.chat.models.chat_model import Chat, ChatRecord, CreateChat, ChatInfo, RenameChat, ChatQuestion, ChatLog, \
     TypeEnum, OperationEnum, ChatRecordResult, ChatLogHistory, ChatLogHistoryItem
+from apps.datasource.crud.datasource import get_ds
 from apps.datasource.crud.recommended_problem import get_datasource_recommended_chart
 from apps.datasource.models.datasource import CoreDatasource
 from apps.db.constant import DB
+from apps.db.db import exec_sql
 from apps.system.crud.assistant import AssistantOutDs, AssistantOutDsFactory
 from apps.system.schemas.system_schema import AssistantOutDsSchema
 from common.core.deps import CurrentAssistant, SessionDep, CurrentUser, Trans
+from common.utils.data_format import DataFormat
 from common.utils.utils import extract_nested_json
 
 
@@ -234,6 +237,20 @@ def get_chart_data_with_user(session: SessionDep, current_user: CurrentUser, cha
             pass
     return {}
 
+def get_chart_data_with_user_live(session: SessionDep, current_user: CurrentUser, chat_record_id: int):
+    stmt = select(ChatRecord.datasource,ChatRecord.sql).where(and_(ChatRecord.id == chat_record_id, ChatRecord.create_by == current_user.id))
+    row = session.execute(stmt).first()
+    return get_chart_data_ds(session,row.datasource, row.sql)
+
+def get_chart_data_ds(session: SessionDep,ds_id,sql):
+    try:
+        datasource = get_ds(session,ds_id)
+        result = exec_sql(ds=datasource,sql=sql, origin_column=False)
+        _data = DataFormat.convert_large_numbers_in_object_array(result.get('data'))
+        return _data
+    except Exception:
+        pass
+    return []
 
 def get_chat_chart_data(session: SessionDep, chat_record_id: int):
     stmt = select(ChatRecord.data).where(and_(ChatRecord.id == chat_record_id))
@@ -307,7 +324,7 @@ def get_chat_with_records(session: SessionDep, chart_id: int, current_user: Curr
     predict_alias_log = aliased(ChatLog)
 
     stmt = (select(ChatRecord.id, ChatRecord.chat_id, ChatRecord.create_time, ChatRecord.finish_time,
-                   ChatRecord.question, ChatRecord.sql_answer, ChatRecord.sql,
+                   ChatRecord.question, ChatRecord.sql_answer, ChatRecord.sql,ChatRecord.datasource,
                    ChatRecord.chart_answer, ChatRecord.chart, ChatRecord.analysis, ChatRecord.predict,
                    ChatRecord.datasource_select_answer, ChatRecord.analysis_record_id, ChatRecord.predict_record_id,
                    ChatRecord.regenerate_record_id,
@@ -334,7 +351,7 @@ def get_chat_with_records(session: SessionDep, chart_id: int, current_user: Curr
         ChatRecord.create_time))
     if with_data:
         stmt = select(ChatRecord.id, ChatRecord.chat_id, ChatRecord.create_time, ChatRecord.finish_time,
-                      ChatRecord.question, ChatRecord.sql_answer, ChatRecord.sql,
+                      ChatRecord.question, ChatRecord.sql_answer, ChatRecord.sql,ChatRecord.datasource,
                       ChatRecord.chart_answer, ChatRecord.chart, ChatRecord.analysis, ChatRecord.predict,
                       ChatRecord.datasource_select_answer, ChatRecord.analysis_record_id, ChatRecord.predict_record_id,
                       ChatRecord.regenerate_record_id,
@@ -400,7 +417,7 @@ def get_chat_with_records(session: SessionDep, chart_id: int, current_user: Curr
                                  finish_time=row.finish_time,
                                  duration=duration,
                                  total_tokens=total_tokens,
-                                 question=row.question, sql_answer=row.sql_answer, sql=row.sql,
+                                 question=row.question, sql_answer=row.sql_answer, sql=row.sql, datasource=row.datasource,
                                  chart_answer=row.chart_answer, chart=row.chart,
                                  analysis=row.analysis, predict=row.predict,
                                  datasource_select_answer=row.datasource_select_answer,
@@ -419,7 +436,7 @@ def get_chat_with_records(session: SessionDep, chart_id: int, current_user: Curr
                                  finish_time=row.finish_time,
                                  duration=duration,
                                  total_tokens=total_tokens,
-                                 question=row.question, sql_answer=row.sql_answer, sql=row.sql,
+                                 question=row.question, sql_answer=row.sql_answer, sql=row.sql, datasource=row.datasource,
                                  chart_answer=row.chart_answer, chart=row.chart,
                                  analysis=row.analysis, predict=row.predict,
                                  datasource_select_answer=row.datasource_select_answer,

backend/apps/chat/models/chat_model.py

@@ -140,6 +140,7 @@ class ChatRecordResult(BaseModel):
     question: Optional[str] = None
     sql_answer: Optional[str] = None
     sql: Optional[str] = None
+    datasource: Optional[int] = None
     data: Optional[str] = None
     chart_answer: Optional[str] = None
     chart: Optional[str] = None

backend/apps/chat/task/llm.py

@@ -1005,6 +1005,7 @@ def save_sql_data(self, session: Session, data_obj: Dict[str, Any]):
                     data_obj['limit'] = limit
                 else:
                     data_obj['data'] = data_result
+                data_obj['datasource'] = self.ds.id
             return save_sql_exec_data(session=session, record_id=self.record.id,
                                       data=orjson.dumps(data_obj).decode())
         except Exception as e:

backend/apps/dashboard/crud/dashboard_service.py

@@ -1,4 +1,9 @@
+import base64
+
+from orjson import orjson
 from sqlalchemy import select, and_, text
+
+from apps.chat.curd.chat import get_chart_data_ds
 from apps.dashboard.models.dashboard_model import CoreDashboard, CreateDashboard, QueryDashboard, DashboardBaseResponse
 from common.core.deps import SessionDep, CurrentUser
 import uuid
@@ -27,7 +32,7 @@ def list_resource(session: SessionDep, dashboard: QueryDashboard, current_user:
     nodes = [DashboardBaseResponse(**row) for row in result.mappings()]
     tree = build_tree_generic(nodes, root_pid="root")
     return tree
-    
+
 
 def load_resource(session: SessionDep, dashboard: QueryDashboard):
     sql = text("""
@@ -41,7 +46,16 @@ def load_resource(session: SessionDep, dashboard: QueryDashboard):
                WHERE cd.id = :dashboard_id
                """)
     result = session.execute(sql, {"dashboard_id": dashboard.id}).mappings().first()
-    return result
+
+    result_dict = dict(result)
+    canvas_view_obj = orjson.loads(result_dict['canvas_view_info'])
+    for item in canvas_view_obj.values():  # 直接遍历值
+        # 检查必要字段是否存在
+        if all(key in item for key in ['datasource', 'sql']) and item['datasource'] is not None:
+            item['data']['data'] = get_chart_data_ds(session, item['datasource'], item['sql'])
+
+    result_dict['canvas_view_info'] = orjson.dumps(canvas_view_obj)
+    return result_dict
 
 
 def get_create_base_info(user: CurrentUser, dashboard: CreateDashboard):

backend/apps/swagger/locales/en.json

@@ -140,6 +140,7 @@
   "get_chat": "Get Chat Details",
   "get_chat_with_data": "Get Chat Details (With Data)",
   "get_chart_data": "Get Chart Data",
+  "get_chart_data_live": "Get Chart Data Live",
   "get_chart_predict_data": "Get Chart Prediction Data",
   "get_record_log": "Get Chart Record Log",
   "get_record_usage": "Get Chart Record Token Usage & Duration",

backend/apps/swagger/locales/zh.json

@@ -140,6 +140,7 @@
   "get_chat": "获取对话详情",
   "get_chat_with_data": "获取对话详情(带数据)",
   "get_chart_data": "获取图表数据",
+  "get_chart_data_live": "获取图表实时数据",
   "get_chart_predict_data": "获取图表预测数据",
   "get_record_log": "获取对话日志",
   "get_record_usage": "获取对话Token使用量及耗时",

frontend/src/api/chat.ts

@@ -39,6 +39,7 @@ export class ChatRecord {
   question?: string
   sql_answer?: string
   sql?: string
+  datasource?: number
   data?: string | any
   chart_answer?: string
   chart?: string
@@ -67,6 +68,7 @@ export class ChatRecord {
     question: string,
     sql_answer: string | undefined,
     sql: string | undefined,
+    datasource: number | undefined,
     data: string | any | undefined,
     chart_answer: string | undefined,
     chart: string | undefined,
@@ -94,6 +96,7 @@ export class ChatRecord {
     question?: string,
     sql_answer?: string,
     sql?: string,
+    datasource?: number | undefined,
     data?: string | any,
     chart_answer?: string,
     chart?: string,
@@ -120,6 +123,7 @@ export class ChatRecord {
     this.question = question
     this.sql_answer = sql_answer
     this.sql = sql
+    this.datasource = datasource
     this.data = data
     this.chart_answer = chart_answer
     this.chart = chart
@@ -263,6 +267,7 @@ const toChatRecord = (data?: any): ChatRecord | undefined => {
     data.question,
     data.sql_answer,
     data.sql,
+    data.datasource,
     data.data,
     data.chart_answer,
     data.chart,

frontend/src/views/chat/chat-block/ChartBlock.vue

@@ -55,6 +55,8 @@ const dataObject = computed<{
   fields: Array<string>
   data: Array<{ [key: string]: any }>
   limit: number | undefined
+  datasource: number | undefined
+  sql: string | undefined
 }>(() => {
   if (props.message?.record?.data) {
     if (typeof props.message?.record?.data === 'string') {
@@ -220,6 +222,8 @@ function addToDashboard() {
     data: {
       data: data.value,
     },
+    sql: props.message?.record?.sql,
+    datasource: props.message?.record?.datasource,
     chart: {},
   }
   // @ts-expect-error eslint-disable-next-line @typescript-eslint/ban-ts-comment

frontend/src/views/dashboard/editor/ChatChartSelection.vue

@@ -143,7 +143,13 @@ function adaptorChartInfoList(chatInfo: ChatInfo) {
           record?.predict_record_id !== null &&
           data?.data?.length > 0)
       ) {
-        const recordeInfo = { id: chatInfo.id + '_' + record.id, data: data, chart: {} }
+        const recordeInfo = {
+          id: chatInfo.id + '_' + record.id,
+          sql: record.sql,
+          datasource: record.datasource,
+          data: data,
+          chart: {},
+        }
         const chartBaseInfo = JSON.parse(record.chart)
         if (chartBaseInfo) {
           let yAxis = []