From dead01129ad5f4d2bfd084b4cae4c47e58aae242 Mon Sep 17 00:00:00 2001 From: Kyle Sammons <1023070+kyle-sammons@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:42:28 -0800 Subject: [PATCH] Fix metadata field rendering and log off-by-one error (#51) * Fix metadata fields accidentally being hidden * Fix off-by-one error * Moved from a hardcoded string to a dynamic one * Refactor into a proper type, fix other instances of hardcoded values --- .../components/FieldValueFrequency.tsx | 5 +- src/datasource/components/Logs/LogsCell.tsx | 62 ++++++++------ src/datasource/components/Logs/LogsTable.tsx | 7 +- src/datasource/components/Logs/LogsView.tsx | 4 +- src/datasource/types.ts | 11 +++ src/pages/explore.tsx | 85 +++++++++++++++---- 6 files changed, 128 insertions(+), 46 deletions(-) diff --git a/src/datasource/components/FieldValueFrequency.tsx b/src/datasource/components/FieldValueFrequency.tsx index 518b988..1d122dc 100644 --- a/src/datasource/components/FieldValueFrequency.tsx +++ b/src/datasource/components/FieldValueFrequency.tsx @@ -6,6 +6,7 @@ import { HorizontalGroup, VerticalGroup, Button } from '@grafana/ui'; interface Props { field: Field; children: JSX.Element; + logMessageField: string; onPlusClick?: (field: Field, value: string) => void; onMinusClick?: (field: Field, value: string) => void; } @@ -130,9 +131,9 @@ const InnerFooter = (field: Field) => { /** * A component to show the FieldValueFrequency for a given field value in the app UI. */ -const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick }: Props) => { +const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick, logMessageField }: Props) => { // This doesn't make sense for this field - if (field.name === '_source') { + if (field.name === logMessageField) { return <>; } diff --git a/src/datasource/components/Logs/LogsCell.tsx b/src/datasource/components/Logs/LogsCell.tsx index 45791ce..3a9e1ee 100644 --- a/src/datasource/components/Logs/LogsCell.tsx +++ b/src/datasource/components/Logs/LogsCell.tsx @@ -26,7 +26,7 @@ const LogKeyVal = ({ field, val }: LogKeyValProps) => {
- {val} + {JSON.stringify(val)}
); } @@ -45,7 +45,7 @@ const ExpandedLogKeyVal = ({ field, val }: ExpandedLogKeyValProps) => { - {val} + {JSON.stringify(val)} ); } @@ -56,10 +56,15 @@ interface ExpandedDocumentProps { datasourceUid: string, datasourceName: string, datasourceField: string, + logMessageField: string, } -const ExpandedDocument = ({ log, index, datasourceUid, datasourceName, datasourceField }: ExpandedDocumentProps) => { +const ExpandedDocument = ({ log, index, datasourceUid, datasourceName, datasourceField, logMessageField }: ExpandedDocumentProps) => { + // The index in the logs is off by one from the index in the table (due to the header row). In this + // case we care about the index in the table, so add one to it. + index += 1; + const { setSize, windowWidth } = getLogTableContext(); const root = React.useRef(); React.useEffect(() => { @@ -115,11 +120,13 @@ const ExpandedDocument = ({ log, index, datasourceUid, datasourceName, datasour { Array.from(log.keys()).map((key) => ( - + key !== logMessageField ? + + : <> )) } @@ -143,7 +150,7 @@ const ExpandedDocument = ({ log, index, datasourceUid, datasourceName, datasour } -const DocumentCell = (log: Log, style: any, rowIndex: number, expanded: boolean, datasourceUid: string, datasourceName: string, datasourceField: string) => ( +const DocumentCell = (log: Log, style: any, rowIndex: number, expanded: boolean, datasourceUid: string, datasourceName: string, datasourceField: string, logMessageField: string) => (
{ Array.from(log.keys()).map((key) => ( - + key !== logMessageField ? + + : <> )) }
@@ -174,6 +183,7 @@ const DocumentCell = (log: Log, style: any, rowIndex: number, expanded: boolean, datasourceUid={datasourceUid} datasourceName={datasourceName} datasourceField={datasourceField} + logMessageField={logMessageField} />) : '' @@ -260,16 +270,14 @@ const shrinkRows = (expandedRows: boolean[], rowIndex: number, setSize: (index: } - const LogCell = ({ columnIndex, rowIndex, style, data }) => { - const log = data.logs[rowIndex]; - const timestamp = data.timestamps[rowIndex]; const column = data.columns[columnIndex]; const setExpandedRowsAndReRender = data.setExpandedRowsAndReRender; const expandedRows = data.expandedRows; const datasourceUid: string = data.datasourceUid; const datasourceName: string = data.datasourceName; const datasourceField: string = data.datasourceField; + const logMessageField: string = data.logMessageField; const { setSize } = getLogTableContext(); const darkModeEnabled = useTheme2().isDark ; @@ -278,12 +286,6 @@ const LogCell = ({ columnIndex, rowIndex, style, data }) => { // const _timeField = data.timeField; // const _setColumns = data.setColumns - const handleOnClick = (rowIndex: number): any => { - const newExpandedRows = invertRow(expandedRows, rowIndex); - shrinkRows(newExpandedRows, rowIndex, setSize); - setExpandedRowsAndReRender([...newExpandedRows], rowIndex); - } - const outline = darkModeEnabled ? DARK_THEME_OUTLINE : LIGHT_THEME_OUTLINE; // Handle drawing the borders for the entire row @@ -310,10 +312,22 @@ const LogCell = ({ columnIndex, rowIndex, style, data }) => { } + // The 0th row is the header row, but we still need to render the data in + // row 0. Thus the rowIndex is technically 1 more than the log length + rowIndex -= 1; + const log = data.logs[rowIndex]; + const timestamp = data.timestamps[rowIndex]; + + const handleOnClick = (rowIndex: number): any => { + const newExpandedRows = invertRow(expandedRows, rowIndex); + shrinkRows(newExpandedRows, rowIndex + 1, setSize); + setExpandedRowsAndReRender([...newExpandedRows], rowIndex); + } + if (column.logColumnType === LogColumnType.TIME) { return TimestampCell(timestamp, style, rowIndex, expandedRows, handleOnClick); } else if (column.logColumnType === LogColumnType.DOCUMENT) { - return DocumentCell(log, style, rowIndex, expandedRows[rowIndex], datasourceUid, datasourceName, datasourceField); + return DocumentCell(log, style, rowIndex, expandedRows[rowIndex], datasourceUid, datasourceName, datasourceField, logMessageField); } else { return FieldCell(); } diff --git a/src/datasource/components/Logs/LogsTable.tsx b/src/datasource/components/Logs/LogsTable.tsx index 7b39e76..6946fef 100644 --- a/src/datasource/components/Logs/LogsTable.tsx +++ b/src/datasource/components/Logs/LogsTable.tsx @@ -33,9 +33,10 @@ interface LogsTableProps { setExpandedRows: ((value: boolean[] | ((preVar: boolean[]) => boolean[])) => void); setColumns: ((value: LogColumn[] | ((preVar: LogColumn[]) => LogColumn[])) => void); datasourceField: string; + logMessageField: string; } -const LogsTable = ({ logs, timeField, columns, timestamps, expandedRows, setColumns, setExpandedRows, datasourceUid, datasourceName, datasourceField }: LogsTableProps) => { +const LogsTable = ({ logs, timeField, columns, timestamps, expandedRows, setColumns, setExpandedRows, datasourceUid, datasourceName, datasourceField, logMessageField }: LogsTableProps) => { let gridRef: React.RefObject = React.createRef(); // In order to get highly variable (and unknown at the time of rendering) row heights in a virtualized environment @@ -76,10 +77,10 @@ const LogsTable = ({ logs, timeField, columns, timestamps, expandedRows, setColu columnCount={columns.length} columnWidth={index => columns[index].logColumnType === LogColumnType.TIME ? 300 : width-310} height={height} - rowCount={logs.length} + rowCount={logs.length + 1} rowHeight={getSize} width={width} - itemData={{logs, timestamps, columns, timeField, setColumns, setExpandedRowsAndReRender, expandedRows, datasourceUid, datasourceName, datasourceField}} + itemData={{logs, timestamps, columns, timeField, setColumns, setExpandedRowsAndReRender, expandedRows, datasourceUid, datasourceName, datasourceField, logMessageField}} > {LogCell} diff --git a/src/datasource/components/Logs/LogsView.tsx b/src/datasource/components/Logs/LogsView.tsx index 7357557..46c3530 100644 --- a/src/datasource/components/Logs/LogsView.tsx +++ b/src/datasource/components/Logs/LogsView.tsx @@ -11,9 +11,10 @@ interface LogsViewProps { datasourceUid: string; datasourceName: string; datasourceField: string; + logMessageField: string; } -const LogsView = ({ logs, timeField, timestamps, datasourceUid, datasourceName, datasourceField }: LogsViewProps) => { +const LogsView = ({ logs, timeField, timestamps, datasourceUid, datasourceName, datasourceField, logMessageField }: LogsViewProps) => { const [columns, setColumns] = React.useState([ { logColumnType: LogColumnType.TIME, @@ -46,6 +47,7 @@ const LogsView = ({ logs, timeField, timestamps, datasourceUid, datasourceName, datasourceUid={datasourceUid} datasourceName={datasourceName} datasourceField={datasourceField} + logMessageField={logMessageField} /> ) diff --git a/src/datasource/types.ts b/src/datasource/types.ts index 8326cac..00e2d21 100644 --- a/src/datasource/types.ts +++ b/src/datasource/types.ts @@ -108,3 +108,14 @@ export interface Field { } export type Log = Map; + +export type DatasourceUserConfig = { + database: string; + flavor: string; + logLevelField: string; + logMessageField: string; + maxConcurrentShardRequests: number; + pplEnabled: boolean; + timeField: string; + version: string; +} diff --git a/src/pages/explore.tsx b/src/pages/explore.tsx index 492ab61..c1c6087 100644 --- a/src/pages/explore.tsx +++ b/src/pages/explore.tsx @@ -37,7 +37,7 @@ import { import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; import { VariableHide } from '@grafana/schema'; -import { Field, Log } from 'datasource/types'; +import { Field, Log, DatasourceUserConfig } from 'datasource/types'; import FieldValueFrequency from '../datasource/components/FieldValueFrequency'; import LogsView from 'datasource/components/Logs/LogsView'; import { FixedSizeList as List } from 'react-window' @@ -74,6 +74,7 @@ interface FieldStatsState extends SceneObjectState { topTenMostPopularFields: Field[]; visible: boolean; loading: boolean; + datasourceUserConfig?: DatasourceUserConfig; } interface LogsState extends SceneObjectState { @@ -82,6 +83,7 @@ interface LogsState extends SceneObjectState { loading: boolean; totalCount: number; totalFailed: number; + datasourceUserConfig?: DatasourceUserConfig; } const NodeStatsRenderer = ({ model }: SceneComponentProps) => { @@ -183,7 +185,7 @@ const KaldbQueryRenderer = ({ model }: SceneComponentProps) => { ); }; -const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { +const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[], datasourceUserConfig: DatasourceUserConfig) => { const getIcon = (field: Field): string => { if (field.type === 'string') { return 'fa fas fa-font'; @@ -259,6 +261,7 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { const ListItem = ({ index, data, style }) => { const field = data.fields[index]; const isTopTenMostPopularField = index <= data.topTenMostPopularFieldsLength; + const logMessageField = data.logMessageField const isDarkTheme = useTheme2().isDark; let fieldBackgroundColor = isDarkTheme ? DARK_THEME_BACKGROUND : LIGHT_THEME_BACKGROUND; @@ -274,6 +277,7 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { queryComponent.appendToQuery(`${field.name}: ${value}`)} onMinusClick={(field: Field, value: string) => queryComponent.appendToQuery(`NOT ${field.name}: ${value}`) @@ -316,7 +320,8 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { itemData={ { fields: [...topTenMostPopularFields, ...fields], - topTenMostPopularFieldsLength: topTenMostPopularFields.length + topTenMostPopularFieldsLength: topTenMostPopularFields.length, + logMessageField: datasourceUserConfig ? datasourceUserConfig.logMessageField : null }} itemSize={30} @@ -337,7 +342,7 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { }; const KalDBFieldsRenderer = ({ model }: SceneComponentProps) => { - const { fields, topTenMostPopularFields, visible, loading } = model.useState(); + const { fields, topTenMostPopularFields, visible, loading, datasourceUserConfig } = model.useState(); const getFoldIcon = () => { if (visible) { @@ -372,7 +377,7 @@ const KalDBFieldsRenderer = ({ model }: SceneComponentProps) => { - {visible ? KalDBFieldsList(fields, topTenMostPopularFields) : null} + {visible ? KalDBFieldsList(fields, topTenMostPopularFields, datasourceUserConfig) : null} )} @@ -387,9 +392,16 @@ class FieldStats extends SceneObjectBase { topTenMostPopularFields: [], visible: true, loading: true, + datasourceUserConfig: null, ...state, }); } + setDatasourceUserConfig= (datasourceUserConfig: DatasourceUserConfig) => { + this.setState({ + datasourceUserConfig: datasourceUserConfig, + }); + }; + setTopTenMostPopularFields = (fields: Field[]) => { this.setState({ topTenMostPopularFields: fields, @@ -416,7 +428,7 @@ class FieldStats extends SceneObjectBase { } const KalDBLogsRenderer = ({ model }: SceneComponentProps) => { - const { logs, loading, timestamps } = model.useState(); + const { logs, loading, timestamps, datasourceUserConfig } = model.useState(); // TODO: This should be whatever the user set const timeField = "_timesinceepoch" @@ -455,6 +467,7 @@ const KalDBLogsRenderer = ({ model }: SceneComponentProps) => { datasourceUid={linkedDatasourceUid} datasourceName={linkedDatasourceName} datasourceField={linkedDatasourceField} + logMessageField={datasourceUserConfig ? datasourceUserConfig.logMessageField : ''} /> )} @@ -471,10 +484,17 @@ class KalDBLogs extends SceneObjectBase { totalCount: 0, totalFailed: 0, timestamps: [], + datasourceUserConfig: null, ...state, }); } + setDatasourceUserConfig = (datasourceUserConfig: DatasourceUserConfig) => { + this.setState({ + datasourceUserConfig: datasourceUserConfig + }); + } + setTimestamps = (timestamps: number[]) => { this.setState({ timestamps: timestamps, @@ -684,27 +704,59 @@ const parseAndExtractLogData = (data: DataFrame[]) => { // Set field names, the most popular fields, and calculates the frequency of the most common values if (data.length > 0 && data[0].fields.length > 0) { + const currentDataSource = dataSourceVariable + ['getDataSourceTypes']() // This is gross, but we need to access this private property and this is the only real typesafe way to do so in TypeScript + .filter((ele) => ele.name === dataSourceVariable.getValueText())[0]; + + let datasourceUserConfig: DatasourceUserConfig = null; + + if (currentDataSource) { + datasourceUserConfig = + { + database: currentDataSource.jsonData.database, + flavor: currentDataSource.jsonData.flavor, + logLevelField: currentDataSource.jsonData.logLevelField, + logMessageField: currentDataSource.jsonData.logMessageField, + maxConcurrentShardRequests: currentDataSource.jsonData.maxConcurrentShardRequests, + pplEnabled: currentDataSource.jsonData.pplEnabled, + timeField: currentDataSource.jsonData.timeField, + version: currentDataSource.jsonData.version, + }; + logsComponent.setDatasourceUserConfig(datasourceUserConfig); + } + let fieldCounts: Map = new Map(); let mappedFields: Map = new Map(); - let reconstructedLogs: Log[] = []; + let reconstructedLogs: Log[] = [...Array(data[0].length)]; let timestamps: number[] = []; for (let unmappedField of data[0].fields) { + // TODO: Ignore the logMessageField (e.g. _source) for now. We'll likely need to revisit this + // when/if we want to support the JSON view + if (unmappedField.name === logsComponent.state.datasourceUserConfig.logMessageField) { + continue + } + let unmappedFieldValuesArray = unmappedField.values.toArray(); let logsWithDefinedValue = unmappedFieldValuesArray.filter((value) => value !== undefined).length; - // TODO: This should be user configurable - if (unmappedField.name === "_timesinceepoch") { + if (unmappedField.name === logsComponent.state.datasourceUserConfig.timeField) { timestamps = [ ...unmappedField.values.toArray() ]; } - if (unmappedField.name === "_source") { - reconstructedLogs = unmappedField.values.toArray().map( - (value: object) => ( - new Map(Object.entries(value)) - )); - continue; - } + + // Convert the dataframe into how we traditionally think of logs + unmappedFieldValuesArray.forEach((value, index) => { + let newMap = new Map(); + if (reconstructedLogs[index] !== undefined) { + newMap = reconstructedLogs[index]; + } + + if (value) { + newMap.set(unmappedField.name, value); + reconstructedLogs[index] = newMap; + } + }) let mapped_field: Field = { name: unmappedField.name, @@ -737,6 +789,7 @@ const parseAndExtractLogData = (data: DataFrame[]) => { fieldComponent.setFields([...mappedFields.values()]); fieldComponent.setTopTenMostPopularFields(topTenMostPopularFields); + fieldComponent.setDatasourceUserConfig(datasourceUserConfig); } return data; }