From 3d4ae11898e404f9bdfa1690684a3a404b214968 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Tue, 16 Jul 2024 21:43:42 -0700 Subject: [PATCH 1/7] add page for running queries --- server/app/header.tsx | 2 +- server/app/page.tsx | 2 +- server/app/query/page.tsx | 202 +++++++++++++++++++++++++ server/app/query/servers.tsx | 39 +++-- server/app/query/view/[id]/page.tsx | 31 +++- server/data/query.ts | 22 +++ sidecar/app/routes/start.py | 4 +- sidecar/tests/app/routes/test_start.py | 6 +- 8 files changed, 277 insertions(+), 31 deletions(-) diff --git a/server/app/header.tsx b/server/app/header.tsx index 33dc5b4..7be10a6 100644 --- a/server/app/header.tsx +++ b/server/app/header.tsx @@ -7,7 +7,7 @@ import { Bars3Icon, BellIcon, XMarkIcon } from "@heroicons/react/24/outline"; import clsx from "clsx"; import beerTap from "@/public/beer-tap.png"; -const navigation = [{ name: "Dashboard", href: "/query", current: true }]; +const navigation = [{ name: "Queries", href: "/query", current: true }]; const userNavigation = [ { name: "Your Profile", href: "#" }, { name: "Settings", href: "#" }, diff --git a/server/app/page.tsx b/server/app/page.tsx index a006c5e..b6d013b 100644 --- a/server/app/page.tsx +++ b/server/app/page.tsx @@ -58,7 +58,7 @@ export default async function Example() {
{isLoggedIn ? ( - Dashboard + Queries ) : ( Log in )} diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index 3da7f61..fca22de 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -1,4 +1,119 @@ +"use client"; + +import { useEffect, useState } from "react"; +import Link from "next/link"; + +import { StatusPill, RunTimePill } from "@/app/query/view/[id]/components"; +import { + Status, + RemoteServer, + RemoteServerNames, + IPARemoteServers, //hack until the queryId is stored in a DB + StatusByRemoteServer, + StartTimeByRemoteServer, + EndTimeByRemoteServer, + initialStatusByRemoteServer, + initialStartTimeByRemoteServer, + initialEndTimeByRemoteServer, +} from "@/app/query/servers"; +import { getQueryByUUID, Query } from "@/data/query"; + +type QueryData = { + status: StatusByRemoteServer; + startTime: StartTimeByRemoteServer; + endTime: EndTimeByRemoteServer; + query: Query; +}; +type DataByQuery = { + [queryID: string]: QueryData; +}; + export default function Page() { + const [queryIDs, setQueryIDs] = useState([]); + const [dataByQuery, setDataByQuery] = useState({}); + + const updateData = ( + query: Query, + remoteServer: RemoteServer, + key: keyof QueryData, + value: Status | number, + ) => { + setDataByQuery((prev) => { + let _prev = prev; + if (!prev.hasOwnProperty(query.uuid)) { + // if queryID not in dataByQuery yet, + // add initial status before updating value + _prev = { + ..._prev, + [query.uuid]: { + status: initialStatusByRemoteServer, + startTime: initialStartTimeByRemoteServer, + endTime: initialEndTimeByRemoteServer, + query: query, + }, + }; + } + + return { + ..._prev, + [query.uuid]: { + ..._prev[query.uuid], + [key]: { + ..._prev[query.uuid][key], + [remoteServer.remoteServerName]: value, + }, + }, + }; + }); + }; + + useEffect(() => { + // poll runningQueries every second + (async () => { + const interval = setInterval(async () => { + const _queryIDs: string[] = + await IPARemoteServers[RemoteServerNames.Helper1].runningQueries(); + + setQueryIDs(_queryIDs); + }, 1000); // 1000 milliseconds = 1 second + return () => clearInterval(interval); + })(); + }, []); + + useEffect(() => { + (async () => { + let webSockets: WebSocket[] = []; + + // remove queries when no longer running + const filteredDataByQuery = Object.fromEntries( + Object.keys(dataByQuery) + .filter((queryID) => queryIDs.includes(queryID)) + .map((queryID) => [queryID, dataByQuery[queryID]]), + ); + setDataByQuery(filteredDataByQuery); + + for (const queryID of queryIDs) { + const query: Query = await getQueryByUUID(queryID); + + for (const remoteServer of Object.values(IPARemoteServers)) { + const statusWs = remoteServer.openStatusSocket( + queryID, + (status) => updateData(query, remoteServer, "status", status), + (startTime) => + updateData(query, remoteServer, "startTime", startTime), + (endTime) => updateData(query, remoteServer, "endTime", endTime), + ); + webSockets = [...webSockets, statusWs]; + } + } + return () => { + for (const ws of webSockets) { + ws.close(); + } + }; + })(); + }, [queryIDs, dataByQuery]); + return ( <>
@@ -6,6 +121,93 @@ export default function Page() {

Current Queries

+ + {Object.entries(dataByQuery).map(([queryID, queryData]) => { + const statusByRemoteServer = queryData.status; + const startTimeByRemoteServer = queryData.startTime; + const endTimeByRemoteServer = queryData.endTime; + const query = queryData.query; + + return ( +
+ +
+

+ Query: {query.displayId} +

+
+
+ {Object.values(IPARemoteServers).map( + (remoteServer: RemoteServer) => { + const startTime = + startTimeByRemoteServer[ + remoteServer.remoteServerName + ]; + const endTime = + endTimeByRemoteServer[ + remoteServer.remoteServerName + ]; + + const status = + statusByRemoteServer[ + remoteServer.remoteServerName + ] ?? Status.UNKNOWN; + + return ( +
+
+ {remoteServer.toString()} Run Time +
+
+ +
+
+ ); + }, + )} +
+
+
+
+ {Object.values(IPARemoteServers).map( + (remoteServer: RemoteServer) => { + const status = + statusByRemoteServer[ + remoteServer.remoteServerName + ] ?? Status.UNKNOWN; + + return ( +
+
+ {remoteServer.remoteServerNameStr} Status +
+
+ +
+
+ ); + }, + )} +
+
+
+ +
+ ); + })}
diff --git a/server/app/query/servers.tsx b/server/app/query/servers.tsx index 610c3f7..6c6e9ef 100644 --- a/server/app/query/servers.tsx +++ b/server/app/query/servers.tsx @@ -96,6 +96,16 @@ export class RemoteServer { throw new Error("Not Implemented"); } + runningQueriesURL(): URL { + return new URL(`/start/running-queries`, this.baseURL); + } + + async runningQueries(): Promise { + const queries_response = await fetch(this.runningQueriesURL()); + const queriesJSON = await queries_response.json(); + return queriesJSON["running_queries"]; + } + logURL(id: string): URL { return new URL(`/start/${id}/log-file`, this.baseURL); } @@ -187,35 +197,22 @@ export class RemoteServer { openStatusSocket( id: string, - setStatus: React.Dispatch>, - setStartTime: React.Dispatch>, - setEndTime: React.Dispatch>, + setStatus: (status: Status) => void, + setStartTime: (startTime: number) => void, + setEndTime: (endTime: number) => void, ): WebSocket { const ws = this.statusSocket(id); const updateStatus = (status: Status) => { - setStatus((prevStatus) => ({ - ...prevStatus, - [this.remoteServerName]: status, - })); + setStatus(status); }; - const updateStartTime = (runTime: number) => { - setStartTime((prevStartTime) => { - return { - ...prevStartTime, - [this.remoteServerName]: runTime, - }; - }); + const updateStartTime = (startTime: number) => { + setStartTime(startTime); }; - const updateEndTime = (runTime: number) => { - setEndTime((prevEndTime) => { - return { - ...prevEndTime, - [this.remoteServerName]: runTime, - }; - }); + const updateEndTime = (endTime: number) => { + setEndTime(endTime); }; ws.onmessage = (event) => { diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 083d5f1..29cbdf7 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -55,6 +55,31 @@ export default function QueryPage({ params }: { params: { id: string } }) { const [endTimeByRemoteServer, setEndTimeByRemoteServer] = useState(initialEndTimeByRemoteServer); + const updateStatus = (remoteServer: RemoteServer, status: Status) => { + setStatusByRemoteServer((prevStatus) => ({ + ...prevStatus, + [remoteServer.remoteServerName]: status, + })); + }; + + const updateStartTime = (remoteServer: RemoteServer, runTime: number) => { + setStartTimeByRemoteServer((prevStartTime) => { + return { + ...prevStartTime, + [remoteServer.remoteServerName]: runTime, + }; + }); + }; + + const updateEndTime = (remoteServer: RemoteServer, runTime: number) => { + setEndTimeByRemoteServer((prevEndTime) => { + return { + ...prevEndTime, + [remoteServer.remoteServerName]: runTime, + }; + }); + }; + function flipLogsHidden() { setLogsHidden(!logsHidden); } @@ -111,9 +136,9 @@ export default function QueryPage({ params }: { params: { id: string } }) { const loggingWs = remoteServer.openLogSocket(query.uuid, setLogs); const statusWs = remoteServer.openStatusSocket( query.uuid, - setStatusByRemoteServer, - setStartTimeByRemoteServer, - setEndTimeByRemoteServer, + (status) => updateStatus(remoteServer, status), + (startTime) => updateStartTime(remoteServer, startTime), + (endTime) => updateEndTime(remoteServer, endTime), ); const statsWs = remoteServer.openStatsSocket( query.uuid, diff --git a/server/data/query.ts b/server/data/query.ts index f199543..4486f87 100644 --- a/server/data/query.ts +++ b/server/data/query.ts @@ -55,6 +55,28 @@ export async function getQuery(displayId: string): Promise { throw new Error(`${displayId} not found.`); } +export async function getQueryByUUID(uuid: string): Promise { + const supabase = await buildSupabaseServerClient(); + + const { status, data, error } = await supabase + .from("queries") + .select("*") + .eq("uuid", uuid) + .limit(1) + .maybeSingle(); + + if (error) { + console.error(error); + } else if (status === 200) { + if (data) { + return processQueryData(data); + } else { + notFound(); + } + } + throw new Error(`${uuid} not found.`); +} + export async function createNewQuery( params: FormData, queryType: QueryType, diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py index 25e8784..6d3b169 100644 --- a/sidecar/app/routes/start.py +++ b/sidecar/app/routes/start.py @@ -25,7 +25,7 @@ class IncorrectRoleError(Exception): pass -@router.get("/capacity_available") +@router.get("/capacity-available") def capacity_available( request: Request, ): @@ -33,7 +33,7 @@ def capacity_available( return {"capacity_available": query_manager.capacity_available} -@router.get("/running_queries") +@router.get("/running-queries") def running_queries( request: Request, ): diff --git a/sidecar/tests/app/routes/test_start.py b/sidecar/tests/app/routes/test_start.py index 8f00236..8ee1aae 100644 --- a/sidecar/tests/app/routes/test_start.py +++ b/sidecar/tests/app/routes/test_start.py @@ -34,20 +34,20 @@ def _running_query(): def test_capacity_available(): - response = client.get("/start/capacity_available") + response = client.get("/start/capacity-available") assert response.status_code == 200 assert response.json() == {"capacity_available": True} def test_not_capacity_available(running_query): assert running_query.query_id in app.state.QUERY_MANAGER.running_queries - response = client.get("/start/capacity_available") + response = client.get("/start/capacity-available") assert response.status_code == 200 assert response.json() == {"capacity_available": False} def test_running_queries(running_query): - response = client.get("/start/running_queries") + response = client.get("/start/running-queries") assert response.status_code == 200 assert response.json() == {"running_queries": [running_query.query_id]} From 801548f3b916faa59415329950320ec4ae8e7c70 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Wed, 17 Jul 2024 15:47:38 -0700 Subject: [PATCH 2/7] refactor to use a StatusEvent interface; update StatusEvent with polling instead of websocket --- server/app/query/page.tsx | 106 +++++++++------------- server/app/query/servers.tsx | 85 ++++++++++------- server/app/query/view/[id]/components.tsx | 29 +++--- server/app/query/view/[id]/page.tsx | 71 +++++---------- sidecar/app/query/ipa.py | 2 +- sidecar/app/routes/start.py | 6 +- sidecar/tests/app/routes/test_start.py | 23 ++++- 7 files changed, 152 insertions(+), 170 deletions(-) diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index fca22de..3262b94 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -5,23 +5,17 @@ import Link from "next/link"; import { StatusPill, RunTimePill } from "@/app/query/view/[id]/components"; import { - Status, + StatusEvent, RemoteServer, RemoteServerNames, IPARemoteServers, //hack until the queryId is stored in a DB - StatusByRemoteServer, - StartTimeByRemoteServer, - EndTimeByRemoteServer, - initialStatusByRemoteServer, - initialStartTimeByRemoteServer, - initialEndTimeByRemoteServer, + StatusEventByRemoteServer, + initialStatusEventByRemoteServer, } from "@/app/query/servers"; import { getQueryByUUID, Query } from "@/data/query"; type QueryData = { - status: StatusByRemoteServer; - startTime: StartTimeByRemoteServer; - endTime: EndTimeByRemoteServer; + statusEvent: StatusEventByRemoteServer; query: Query; }; type DataByQuery = { @@ -35,8 +29,7 @@ export default function Page() { const updateData = ( query: Query, remoteServer: RemoteServer, - key: keyof QueryData, - value: Status | number, + statusEvent: StatusEvent, ) => { setDataByQuery((prev) => { let _prev = prev; @@ -46,9 +39,7 @@ export default function Page() { _prev = { ..._prev, [query.uuid]: { - status: initialStatusByRemoteServer, - startTime: initialStartTimeByRemoteServer, - endTime: initialEndTimeByRemoteServer, + statusEvent: initialStatusEventByRemoteServer, query: query, }, }; @@ -58,9 +49,9 @@ export default function Page() { ..._prev, [query.uuid]: { ..._prev[query.uuid], - [key]: { - ..._prev[query.uuid][key], - [remoteServer.remoteServerName]: value, + statusEvent: { + ..._prev[query.uuid].statusEvent, + [remoteServer.remoteServerName]: statusEvent, }, }, }; @@ -82,37 +73,25 @@ export default function Page() { useEffect(() => { (async () => { - let webSockets: WebSocket[] = []; - - // remove queries when no longer running - const filteredDataByQuery = Object.fromEntries( - Object.keys(dataByQuery) - .filter((queryID) => queryIDs.includes(queryID)) - .map((queryID) => [queryID, dataByQuery[queryID]]), - ); - setDataByQuery(filteredDataByQuery); + setDataByQuery((prev) => { + return Object.fromEntries( + Object.keys(prev) + .filter((queryID) => queryIDs.includes(queryID)) + .map((queryID) => [queryID, prev[queryID]]), + ); + }); for (const queryID of queryIDs) { const query: Query = await getQueryByUUID(queryID); for (const remoteServer of Object.values(IPARemoteServers)) { - const statusWs = remoteServer.openStatusSocket( - queryID, - (status) => updateData(query, remoteServer, "status", status), - (startTime) => - updateData(query, remoteServer, "startTime", startTime), - (endTime) => updateData(query, remoteServer, "endTime", endTime), - ); - webSockets = [...webSockets, statusWs]; + const statusEvent: StatusEvent = + await remoteServer.queryStatus(queryID); + updateData(query, remoteServer, statusEvent); } } - return () => { - for (const ws of webSockets) { - ws.close(); - } - }; })(); - }, [queryIDs, dataByQuery]); + }, [queryIDs]); return ( <> @@ -122,10 +101,14 @@ export default function Page() { Current Queries + {Object.keys(dataByQuery).length == 0 && ( +

+ None currently running. +

+ )} + {Object.entries(dataByQuery).map(([queryID, queryData]) => { - const statusByRemoteServer = queryData.status; - const startTimeByRemoteServer = queryData.startTime; - const endTimeByRemoteServer = queryData.endTime; + const statusEventByRemoteServer = queryData.statusEvent; const query = queryData.query; return ( @@ -142,19 +125,16 @@ export default function Page() {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const startTime = - startTimeByRemoteServer[ + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[ remoteServer.remoteServerName ]; - const endTime = - endTimeByRemoteServer[ - remoteServer.remoteServerName - ]; - - const status = - statusByRemoteServer[ - remoteServer.remoteServerName - ] ?? Status.UNKNOWN; + if (statusEvent === null) { + return <>; + } + const status = statusEvent.status; + const startTime = statusEvent.startTime; + const endTime = statusEvent.endTime; return (
- +
); @@ -181,10 +157,14 @@ export default function Page() {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const status = - statusByRemoteServer[ + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[ remoteServer.remoteServerName - ] ?? Status.UNKNOWN; + ]; + if (statusEvent === null) { + return <>; + } + const status = statusEvent.status; return (
[[serverName], null]), ); -export type StatsByRemoteServer = { - [key in RemoteServerNames]: StatsDataPoint[]; -}; - -export const initialStatsByRemoteServer: StatsByRemoteServer = - Object.fromEntries( - Object.values(RemoteServerNames).map((serverName) => [[serverName], []]), - ); - export type StartTimeByRemoteServer = { [key in RemoteServerNames]: number | null; }; @@ -67,6 +66,10 @@ export type EndTimeByRemoteServer = { [key in RemoteServerNames]: number | null; }; +export type StatusEventByRemoteServer = { + [key in RemoteServerNames]: StatusEvent | null; +}; + export const initialStartTimeByRemoteServer: StartTimeByRemoteServer = Object.fromEntries( Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), @@ -77,6 +80,26 @@ export const initialEndTimeByRemoteServer: StartTimeByRemoteServer = Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), ); +export const initialStatusEventByRemoteServer: StatusEventByRemoteServer = + Object.fromEntries( + Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), + ); + +export interface StatsDataPoint { + timestamp: string; + memoryRSSUsage: number; + cpuUsage: number; +} + +export type StatsByRemoteServer = { + [key in RemoteServerNames]: StatsDataPoint[]; +}; + +export const initialStatsByRemoteServer: StatsByRemoteServer = + Object.fromEntries( + Object.values(RemoteServerNames).map((serverName) => [[serverName], []]), + ); + export class RemoteServer { protected baseURL: URL; remoteServerName: RemoteServerNames; @@ -96,6 +119,16 @@ export class RemoteServer { throw new Error("Not Implemented"); } + queryStatusURL(id: string): URL { + return new URL(`/start/${id}/status`, this.baseURL); + } + + async queryStatus(id: string): Promise { + const status_response = await fetch(this.queryStatusURL(id)); + const statusJSON = await status_response.json(); + return buildStatusEventFromJSON(statusJSON); + } + runningQueriesURL(): URL { return new URL(`/start/running-queries`, this.baseURL); } @@ -103,7 +136,7 @@ export class RemoteServer { async runningQueries(): Promise { const queries_response = await fetch(this.runningQueriesURL()); const queriesJSON = await queries_response.json(); - return queriesJSON["running_queries"]; + return queriesJSON.running_queries; } logURL(id: string): URL { @@ -197,31 +230,13 @@ export class RemoteServer { openStatusSocket( id: string, - setStatus: (status: Status) => void, - setStartTime: (startTime: number) => void, - setEndTime: (endTime: number) => void, + setStatusEvent: (statusEvent: StatusEvent) => void, ): WebSocket { const ws = this.statusSocket(id); - const updateStatus = (status: Status) => { - setStatus(status); - }; - - const updateStartTime = (startTime: number) => { - setStartTime(startTime); - }; - - const updateEndTime = (endTime: number) => { - setEndTime(endTime); - }; - ws.onmessage = (event) => { - const eventData = JSON.parse(event.data); - updateStartTime(eventData.start_time); - updateEndTime(eventData.end_time ?? null); - const statusString: string = eventData.status; - const status = getStatusFromString(statusString); - updateStatus(status); + const statusEvent = buildStatusEventFromJSON(JSON.parse(event.data)); + setStatusEvent(statusEvent); }; ws.onclose = (event) => { diff --git a/server/app/query/view/[id]/components.tsx b/server/app/query/view/[id]/components.tsx index 5aaf35a..2411ddd 100644 --- a/server/app/query/view/[id]/components.tsx +++ b/server/app/query/view/[id]/components.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useRef } from "react"; import { Source_Code_Pro } from "next/font/google"; import clsx from "clsx"; import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/24/solid"; -import { Status, ServerLog } from "@/app/query/servers"; +import { Status, StatusEvent, ServerLog } from "@/app/query/servers"; const sourceCodePro = Source_Code_Pro({ subsets: ["latin"] }); @@ -64,15 +64,7 @@ function secondsToTime(e: number) { return h + ":" + m + ":" + s; } -export function RunTimePill({ - status, - startTime, - endTime, -}: { - status: Status; - startTime: number | null; - endTime: number | null; -}) { +export function RunTimePill({ statusEvent }: { statusEvent: StatusEvent }) { const [runTime, setRunTime] = useState(null); const runTimeStr = runTime ? secondsToTime(runTime) : "N/A"; const intervalId = useRef | null>(null); @@ -83,22 +75,27 @@ export function RunTimePill({ // which runs the timer. if a new one is needed, it's created. clearTimeout(intervalId.current); } - if (startTime === null) { + if (statusEvent?.startTime === null) { setRunTime(null); } else { - if (endTime !== null) { - setRunTime(endTime - startTime); + if (statusEvent?.endTime !== null) { + setRunTime(statusEvent.endTime - statusEvent.startTime); } else { let newIntervalId = setInterval(() => { - setRunTime(Date.now() / 1000 - startTime); + setRunTime(Date.now() / 1000 - statusEvent.startTime); }, 1000); intervalId.current = newIntervalId; } } - }, [startTime, endTime]); + }, [statusEvent]); return ( -
+
{runTimeStr}
); diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 29cbdf7..02cfbc0 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -9,6 +9,7 @@ import { } from "@/app/query/view/[id]/components"; import { Status, + StatusEvent, ServerLog, RemoteServer, RemoteServerNames, @@ -18,10 +19,12 @@ import { StatsByRemoteServer, StartTimeByRemoteServer, EndTimeByRemoteServer, + StatusEventByRemoteServer, initialStatusByRemoteServer, initialStatsByRemoteServer, initialStartTimeByRemoteServer, initialEndTimeByRemoteServer, + initialStatusEventByRemoteServer, } from "@/app/query/servers"; import { StatsComponent } from "@/app/query/view/[id]/charts"; import { JSONSafeParse } from "@/app/utils"; @@ -46,40 +49,21 @@ export default function QueryPage({ params }: { params: { id: string } }) { selectedRemoteServerLogs.includes(item.remoteServer.remoteServerNameStr), ); - const [statusByRemoteServer, setStatusByRemoteServer] = - useState(initialStatusByRemoteServer); const [statsByRemoteServer, setStatsByRemoteServer] = useState(initialStatsByRemoteServer); - const [startTimeByRemoteServer, setStartTimeByRemoteServer] = - useState(initialStartTimeByRemoteServer); - const [endTimeByRemoteServer, setEndTimeByRemoteServer] = - useState(initialEndTimeByRemoteServer); + const [statusEventByRemoteServer, setStatusEventByRemoteServer] = + useState(initialStatusEventByRemoteServer); - const updateStatus = (remoteServer: RemoteServer, status: Status) => { - setStatusByRemoteServer((prevStatus) => ({ + const updateStatusEvent = ( + remoteServer: RemoteServer, + statusEvent: StatusEvent, + ) => { + setStatusEventByRemoteServer((prevStatus) => ({ ...prevStatus, - [remoteServer.remoteServerName]: status, + [remoteServer.remoteServerName]: statusEvent, })); }; - const updateStartTime = (remoteServer: RemoteServer, runTime: number) => { - setStartTimeByRemoteServer((prevStartTime) => { - return { - ...prevStartTime, - [remoteServer.remoteServerName]: runTime, - }; - }); - }; - - const updateEndTime = (remoteServer: RemoteServer, runTime: number) => { - setEndTimeByRemoteServer((prevEndTime) => { - return { - ...prevEndTime, - [remoteServer.remoteServerName]: runTime, - }; - }); - }; - function flipLogsHidden() { setLogsHidden(!logsHidden); } @@ -136,9 +120,7 @@ export default function QueryPage({ params }: { params: { id: string } }) { const loggingWs = remoteServer.openLogSocket(query.uuid, setLogs); const statusWs = remoteServer.openStatusSocket( query.uuid, - (status) => updateStatus(remoteServer, status), - (startTime) => updateStartTime(remoteServer, startTime), - (endTime) => updateEndTime(remoteServer, endTime), + (statusEvent) => updateStatusEvent(remoteServer, statusEvent), ); const statsWs = remoteServer.openStatsSocket( query.uuid, @@ -242,14 +224,11 @@ export default function QueryPage({ params }: { params: { id: string } }) {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const startTime = - startTimeByRemoteServer[remoteServer.remoteServerName]; - const endTime = - endTimeByRemoteServer[remoteServer.remoteServerName]; - - const status = - statusByRemoteServer[remoteServer.remoteServerName] ?? - Status.UNKNOWN; + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[remoteServer.remoteServerName]; + if (statusEvent === null) { + return <>; + } return (
- +
); @@ -298,9 +273,11 @@ export default function QueryPage({ params }: { params: { id: string } }) {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const status = - statusByRemoteServer[remoteServer.remoteServerName] ?? - Status.UNKNOWN; + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[remoteServer.remoteServerName]; + if (statusEvent === null) { + return <>; + } return (
- +
); diff --git a/sidecar/app/query/ipa.py b/sidecar/app/query/ipa.py index eb980a5..044c410 100644 --- a/sidecar/app/query/ipa.py +++ b/sidecar/app/query/ipa.py @@ -251,7 +251,7 @@ def run(self): for sidecar_url in sidecar_urls: url = urlunparse( sidecar_url._replace( - scheme="https", path=f"/start/ipa-helper/{self.query_id}/status" + scheme="https", path=f"/start/{self.query_id}/status" ), ) while True: diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py index 6d3b169..e65f093 100644 --- a/sidecar/app/routes/start.py +++ b/sidecar/app/routes/start.py @@ -109,13 +109,13 @@ def start_ipa_helper( return {"message": "Process started successfully", "query_id": query_id} -@router.get("/ipa-helper/{query_id}/status") -def get_ipa_helper_status( +@router.get("/{query_id}/status") +def get_query_status( query_id: str, request: Request, ): query = get_query_from_query_id(request.app.state.QUERY_MANAGER, Query, query_id) - return {"status": query.status.name} + return query.status_event_json @router.get("/{query_id}/log-file") diff --git a/sidecar/tests/app/routes/test_start.py b/sidecar/tests/app/routes/test_start.py index 8ee1aae..e7e0478 100644 --- a/sidecar/tests/app/routes/test_start.py +++ b/sidecar/tests/app/routes/test_start.py @@ -130,16 +130,29 @@ def test_start_ipa_query_as_helper(mock_role): ) -def test_get_ipa_helper_status_not_found(): +def test_get_status_not_found(): query_id = str(uuid4()) - response = client.get(f"/start/ipa-helper/{query_id}/status") + response = client.get(f"/start/{query_id}/status") assert response.status_code == 404 -def test_get_ipa_helper_status(running_query): - response = client.get(f"/start/ipa-helper/{running_query.query_id}/status") +def test_get_status_running(running_query): + response = client.get(f"/start/{running_query.query_id}/status") assert response.status_code == 200 - assert response.json() == {"status": str(Status.STARTING.name)} + status_event_json = response.json() + assert status_event_json["status"] == str(Status.STARTING.name) + assert "start_time" in status_event_json + assert "end_time" not in status_event_json + + +def test_get_status_complete(running_query): + running_query.status = Status.COMPLETE + response = client.get(f"/start/{running_query.query_id}/status") + assert response.status_code == 200 + status_event_json = response.json() + assert status_event_json["status"] == str(Status.COMPLETE.name) + assert "start_time" in status_event_json + assert "end_time" in status_event_json def test_get_ipa_helper_log_file_not_found(): From 0c485680dbe2924a1ba2371592262d15f8ea6058 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Wed, 17 Jul 2024 16:25:44 -0700 Subject: [PATCH 3/7] small cleanup --- server/app/query/page.tsx | 8 ++------ server/app/query/view/[id]/page.tsx | 7 ------- server/data/query.ts | 6 ++++++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index 3262b94..1a6b558 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -33,7 +33,7 @@ export default function Page() { ) => { setDataByQuery((prev) => { let _prev = prev; - if (!prev.hasOwnProperty(query.uuid)) { + if (!Object.hasOwn(prev, query.uuid)) { // if queryID not in dataByQuery yet, // add initial status before updating value _prev = { @@ -132,9 +132,6 @@ export default function Page() { if (statusEvent === null) { return <>; } - const status = statusEvent.status; - const startTime = statusEvent.startTime; - const endTime = statusEvent.endTime; return (
; } - const status = statusEvent.status; return (
- +
); diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 02cfbc0..6a00410 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -8,22 +8,15 @@ import { LogViewer, } from "@/app/query/view/[id]/components"; import { - Status, StatusEvent, ServerLog, RemoteServer, RemoteServerNames, RemoteServersType, IPARemoteServers, //hack until the queryId is stored in a DB - StatusByRemoteServer, StatsByRemoteServer, - StartTimeByRemoteServer, - EndTimeByRemoteServer, StatusEventByRemoteServer, - initialStatusByRemoteServer, initialStatsByRemoteServer, - initialStartTimeByRemoteServer, - initialEndTimeByRemoteServer, initialStatusEventByRemoteServer, } from "@/app/query/servers"; import { StatsComponent } from "@/app/query/view/[id]/charts"; diff --git a/server/data/query.ts b/server/data/query.ts index 4486f87..289abf9 100644 --- a/server/data/query.ts +++ b/server/data/query.ts @@ -45,6 +45,9 @@ export async function getQuery(displayId: string): Promise { if (error) { console.error(error); + throw new Error( + `Error fetching query with displayId= ${displayId}: ${error.message}`, + ); } else if (status === 200) { if (data) { return processQueryData(data); @@ -67,6 +70,9 @@ export async function getQueryByUUID(uuid: string): Promise { if (error) { console.error(error); + throw new Error( + `Error fetching query with UUID= ${uuid}: ${error.message}`, + ); } else if (status === 200) { if (data) { return processQueryData(data); From 73370e972e51a063f2a9b93dbcef108ae3379da5 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Thu, 18 Jul 2024 22:37:22 -0700 Subject: [PATCH 4/7] remove log polluter in websocket --- sidecar/app/routes/websockets.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sidecar/app/routes/websockets.py b/sidecar/app/routes/websockets.py index d846841..64cf149 100644 --- a/sidecar/app/routes/websockets.py +++ b/sidecar/app/routes/websockets.py @@ -39,11 +39,9 @@ async def status_websocket( async with use_websocket(websocket) as websocket: while query.running: - query.logger.debug(f"{query_id=} Status: {query.status.name}") await websocket.send_json(query.status_event_json) await asyncio.sleep(1) - query.logger.debug(f"{query_id=} Status: {query.status.name}") await websocket.send_json(query.status_event_json) From b0d667b18d0e297de3292ad1c2c6c14aa5990985 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Thu, 18 Jul 2024 23:08:34 -0700 Subject: [PATCH 5/7] fix elements needing key warning --- server/app/query/page.tsx | 8 ++++++-- server/app/query/view/[id]/page.tsx | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index 1a6b558..a2e8668 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -130,7 +130,9 @@ export default function Page() { remoteServer.remoteServerName ]; if (statusEvent === null) { - return <>; + return ( +
+ ); } return ( @@ -159,7 +161,9 @@ export default function Page() { remoteServer.remoteServerName ]; if (statusEvent === null) { - return <>; + return ( +
+ ); } return ( diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 6a00410..024d8a3 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -220,7 +220,7 @@ export default function QueryPage({ params }: { params: { id: string } }) { const statusEvent: StatusEvent | null = statusEventByRemoteServer[remoteServer.remoteServerName]; if (statusEvent === null) { - return <>; + return
; } return ( @@ -269,7 +269,7 @@ export default function QueryPage({ params }: { params: { id: string } }) { const statusEvent: StatusEvent | null = statusEventByRemoteServer[remoteServer.remoteServerName]; if (statusEvent === null) { - return <>; + return
; } return ( From ac6223b0b37a464c917d1bce19f4cbacf567f82a Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Thu, 18 Jul 2024 23:10:27 -0700 Subject: [PATCH 6/7] set runTime before the first interval --- server/app/query/view/[id]/components.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app/query/view/[id]/components.tsx b/server/app/query/view/[id]/components.tsx index 2411ddd..6e43b9b 100644 --- a/server/app/query/view/[id]/components.tsx +++ b/server/app/query/view/[id]/components.tsx @@ -81,6 +81,7 @@ export function RunTimePill({ statusEvent }: { statusEvent: StatusEvent }) { if (statusEvent?.endTime !== null) { setRunTime(statusEvent.endTime - statusEvent.startTime); } else { + setRunTime(Date.now() / 1000 - statusEvent.startTime); let newIntervalId = setInterval(() => { setRunTime(Date.now() / 1000 - statusEvent.startTime); }, 1000); From cfcb502f1aa2028126ce41abe461daa73b1b1f78 Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Thu, 18 Jul 2024 23:15:59 -0700 Subject: [PATCH 7/7] use Promise.all for fetching statusEvents concurrently across remoteServers; update comment in StasusEvent updater --- server/app/query/page.tsx | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index a2e8668..40a9d8c 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -35,7 +35,10 @@ export default function Page() { let _prev = prev; if (!Object.hasOwn(prev, query.uuid)) { // if queryID not in dataByQuery yet, - // add initial status before updating value + // add initial status before updating value. + // otherwise prev[query.uuid][statusEvent][remteServer.ServerName] + // doesn't exist, and cannot be updated. we need to fill in the + // nested structure, which `initialStatusEventByRemoteServer` does. _prev = { ..._prev, [query.uuid]: { @@ -81,15 +84,18 @@ export default function Page() { ); }); - for (const queryID of queryIDs) { + const promises = queryIDs.map(async (queryID) => { const query: Query = await getQueryByUUID(queryID); - - for (const remoteServer of Object.values(IPARemoteServers)) { - const statusEvent: StatusEvent = - await remoteServer.queryStatus(queryID); - updateData(query, remoteServer, statusEvent); - } - } + const remoteServerPromises = Object.values(IPARemoteServers).map( + async (remoteServer) => { + const statusEvent: StatusEvent = + await remoteServer.queryStatus(queryID); + updateData(query, remoteServer, statusEvent); + }, + ); + await Promise.all(remoteServerPromises); + }); + await Promise.all(promises); })(); }, [queryIDs]);