diff --git a/src/components/QueryEditor/QueryBuilder/Editors/ExactValueEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/ExactValueEditor.tsx new file mode 100644 index 00000000..6fa2badd --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/ExactValueEditor.tsx @@ -0,0 +1,32 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps, toOption } from '@grafana/plugin-ui'; +import { InlineField, Select } from '@grafana/ui'; + +import { getFieldValueOptions } from './utils/editorHelper'; + +export default function ExactValueEditor(props: QueryBuilderOperationParamEditorProps) { + const { onChange, index, value, operation } = props; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldValueOptions(props, operation.params[0] as string)); + setIsLoading(false); + }; + return ( + + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => onChange(index, value)} + value={toOption(String(value || ""))} + width="auto" + /> + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/FieldAsFieldEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/FieldAsFieldEditor.tsx new file mode 100644 index 00000000..5eb6c3c2 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/FieldAsFieldEditor.tsx @@ -0,0 +1,85 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps, toOption } from '@grafana/plugin-ui'; +import { InlineField, Stack, Select } from '@grafana/ui'; + +import { isValue, quoteString, getValue } from '../utils/stringHandler'; +import { splitString } from '../utils/stringSplitter'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +export default function FieldAsFieldEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const str = splitString(String(value || "")); + let parsedFromField = ""; + let parsedToField = ""; + if (str.length === 3) { + if (isValue(str[0])) { + parsedFromField = getValue(str[0]); + } + if (str[1].type === "space" && str[1].value === "as") { + if (isValue(str[2])) { + parsedToField = getValue(str[2]); + } + } + } + + const [fromField, setFromField] = useState(parsedFromField); + const [toField, setToField] = useState(parsedToField); + + const updateValue = (fromField: string, toField: string) => { + const parts = [fromField, toField].map(field => { + const trimmed = field.trim(); + return trimmed === "" ? "\"\"" : quoteString(trimmed); + }); + const value = parts.join(" as "); + onChange(index, value); + }; + + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldNameOptions(props)); + setIsLoading(false); + }; + + return ( + + + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => { + setFromField(value); + updateValue(value, toField); + }} + value={toOption(fromField)} + width="auto" + /> + +
as
+ + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => { + setToField(value); + updateValue(fromField, value); + }} + value={toOption(toField)} + width="auto" + /> + + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/FieldEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/FieldEditor.tsx new file mode 100644 index 00000000..fb2bf8d1 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/FieldEditor.tsx @@ -0,0 +1,34 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps, toOption } from '@grafana/plugin-ui'; +import { InlineField, Select } from '@grafana/ui'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +export default function FieldEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldNameOptions(props)); + setIsLoading(false); + } + + return ( + + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => onChange(index, value)} + value={toOption(String(value || ""))} + width="auto" + /> + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/FieldValueTypeEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/FieldValueTypeEditor.tsx new file mode 100644 index 00000000..ee93c207 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/FieldValueTypeEditor.tsx @@ -0,0 +1,32 @@ +import React, { useState } from "react"; + +import { SelectableValue } from "@grafana/data"; +import { QueryBuilderOperationParamEditorProps, toOption } from "@grafana/plugin-ui"; +import { Select } from "@grafana/ui"; + +import { getValueTypeOptions } from "./utils/editorHelper"; + +export default function FieldValueTypeEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getValueTypeOptions(props)); + setIsLoading(false); + } + + return ( + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => onChange(index, value)} + value={toOption(String(value || ""))} + width="auto" + /> + ) +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditor.tsx new file mode 100644 index 00000000..bc1831c1 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditor.tsx @@ -0,0 +1,51 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { ActionMeta, MultiSelect } from '@grafana/ui'; + +import { getValuesFromBrackets } from '../utils/operationParser'; +import { quoteString } from '../utils/stringHandler'; +import { splitString } from '../utils/stringSplitter'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +export default function FieldsEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const setFields = (values: SelectableValue[], action: ActionMeta) => { + let rawValues = values.map(({ value = "" }) => value.trim()).filter(Boolean); + if (action) { + if (action.action === "remove-value") { + const oldValues = getValuesFromBrackets(splitString(String(value || ""))); + rawValues = oldValues.filter((v) => v !== (action.removedValue as SelectableValue).value); + } + } + const newValue = rawValues.map((v) => quoteString(v)).join(", "); + onChange(index, newValue); + } + + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldNameOptions(props)); + setIsLoading(false); + } + + return ( + + onChange={setFields} + options={options} + value={getValuesFromBrackets(splitString(String(value || "")))} + isLoading={isLoading} + allowCustomValue + allowCreateWhileLoading + noOptionsMessage="No labels found" + loadingMessage="Loading labels" + width={20} + onOpenMenu={handleOpenMenu} + /> + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditorWithPrefix.tsx b/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditorWithPrefix.tsx new file mode 100644 index 00000000..2925e94e --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/FieldsEditorWithPrefix.tsx @@ -0,0 +1,133 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { ActionMeta, FormatOptionLabelMeta, MultiSelect } from '@grafana/ui'; + +import { quoteString, unquoteString } from '../utils/stringHandler'; +import { splitByUnescapedChar, SplitString, splitString } from '../utils/stringSplitter'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +interface FieldWithPrefix { + name: string; + isPrefix: boolean; +} + +export default function FieldsEditorWithPrefix(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const str = splitString(String(value || "")); + const parsedValues = parseInputValues(str); + const [values, setValues] = useState(parsedValues); + + const setFields = (newValues: FieldWithPrefix[], action?: ActionMeta) => { + if (action) { + if (action.action === "remove-value") { + newValues = values.filter((v) => v.name !== (action.removedValue as any).name); + } + } + setValues(newValues); + const newValue = newValues.map((field) => { + if (field.isPrefix !== undefined) { + return field.isPrefix ? `${quoteString(field.name)}*` : `${quoteString(field.name)}`; + } + return quoteString(field as unknown as string); + }).join(', '); + onChange(index, newValue); + } + + const togglePrefix = (index: number) => { + const field = values[index]; + field.isPrefix = !field.isPrefix; + setFields(values); + }; + + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const formatOptionLabel = (option: SelectableValue, { context }: FormatOptionLabelMeta) => { + if (context === 'value') { + const field = option as FieldWithPrefix; + const handleToggle = (e: React.SyntheticEvent) => { + e.stopPropagation(); + const idx = values.findIndex((v) => (v).name === field.name); + if (idx !== -1) { + togglePrefix(idx); + } + }; + return ( + + {formatFieldLabel(field)} + + ); + } + return <>{option.label}; + } + + const handleOpenMenu = async () => { + setIsLoading(true); + let options = await getFieldNameOptions(props); + const selectedNames = values.map(v => v.name); + options = options.filter((opt: SelectableValue) => opt.value && !selectedNames.includes(opt.value)); + setOptions(options); + setIsLoading(false); + } + + const handleChange = (values: SelectableValue[], action: ActionMeta) => { + setFields(values.map((v) => v.value || v as FieldWithPrefix), action) + } + + return ( + + openMenuOnFocus + onOpenMenu={handleOpenMenu} + isLoading={isLoading} + allowCustomValue + allowCreateWhileLoading + noOptionsMessage="No labels found" + loadingMessage="Loading labels" + options={options} + value={values} + onChange={handleChange} + formatOptionLabel={formatOptionLabel} + /> + ); +} + +const formatFieldLabel = (field: FieldWithPrefix): string => { + return field.isPrefix ? `${field.name} *` : field.name; +}; + +const parseValue = (value: SplitString[]): FieldWithPrefix => { + if (value.length === 0 || value[0].type === "bracket") { + return { name: '', isPrefix: false }; + } + + if (value[0].type === "quote") { + const { type, value: secondValue } = value[1] || {}; + const isPrefix = type === "space" && secondValue === "*"; + return { name: unquoteString(value[0].value), isPrefix }; + } + + const fieldValue = value[0].value; + const isPrefix = fieldValue.endsWith("*"); + return { + name: isPrefix ? fieldValue.slice(0, -1) : fieldValue, + isPrefix, + }; +} + +const parseInputValues = (str: SplitString[]): FieldWithPrefix[] => { + let fields: FieldWithPrefix[] = []; + for (const field of splitByUnescapedChar(str, ',')) { + if (field.length > 0 && field[0].type !== "bracket") { + fields.push(parseValue(field)); + } + } + return fields; +}; diff --git a/src/components/QueryEditor/QueryBuilder/Editors/LogicalFilterEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/LogicalFilterEditor.tsx new file mode 100644 index 00000000..ad587018 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/LogicalFilterEditor.tsx @@ -0,0 +1,74 @@ +import { css } from "@emotion/css"; +import React, { useMemo } from "react"; + +import { GrafanaTheme2 } from "@grafana/data"; +import { OperationList, QueryBuilderOperationParamEditorProps } from "@grafana/plugin-ui"; +import { InlineField, useStyles2 } from "@grafana/ui"; + +import { VisualQuery } from "../../../../types"; +import { parseExprToVisualQuery, createQueryModellerWithDefaultField } from "../QueryModeller"; +import { VictoriaLogsQueryOperationCategory } from "../VictoriaLogsQueryOperationCategory"; + +function isObj(v: unknown): v is { expr: string; visQuery: VisualQuery, fieldName: string } { + return !!v && typeof v === "object" && "expr" in (v as any) && "visQuery" in (v as any) && "fieldName" in (v as any); +} + +export default function LogicalFilterEditor(props: QueryBuilderOperationParamEditorProps) { + const styles = useStyles2(getStyles); + const { value, onChange, index, datasource, timeRange, onRunQuery, operation } = props; + const fieldName = operation.params[0] as string; + + const queryModeller = useMemo(() => createQueryModellerWithDefaultField(fieldName, [VictoriaLogsQueryOperationCategory.Filters, VictoriaLogsQueryOperationCategory.Operators]), [fieldName]); + + const valueIsObj = isObj(value); + const expr = valueIsObj ? value.expr : String(value ?? ""); + const visQuery = valueIsObj ? value.visQuery : parseExprToVisualQuery(expr, fieldName, queryModeller).query; + const prevFieldsName = valueIsObj ? value.fieldName : fieldName; + + if (prevFieldsName !== fieldName) { // change all defaultFields in the visQuery to the new fieldName + const oldVisQueryOps = visQuery.operations.map((v) => ({ ...v, disabled: false })); // render all operations even when disabled + const oldQueryModeller = createQueryModellerWithDefaultField(prevFieldsName, [VictoriaLogsQueryOperationCategory.Filters, VictoriaLogsQueryOperationCategory.Operators]); + const oldExpr = oldQueryModeller.renderQuery({ operations: oldVisQueryOps }); // old so that old defaultField doesn't get rendered + const newVisQuery = parseExprToVisualQuery(oldExpr, fieldName, queryModeller).query; + for (let i = 0; i < visQuery.operations.length; i++) { + const op = visQuery.operations[i]; + if (op.disabled) { + newVisQuery.operations[i].disabled = op.disabled; // keep disabled operations disabled + } + } + onChange(index, { expr, visQuery: newVisQuery, fieldName } as unknown as string); + }; + + const onVisQueryChange = (visQuery: VisualQuery) => { + const expr = queryModeller.renderQuery(visQuery); + const next = { expr, visQuery, fieldName }; + onChange(index, next as unknown as string); + }; + + return ( + + <> + +
+

+ {expr} +

+ + + ) +} + +const getStyles = (theme: GrafanaTheme2) => { + return { + previewText: css` + font-size: ${theme.typography.bodySmall.fontSize}; + ` + }; +}; diff --git a/src/components/QueryEditor/QueryBuilder/Editors/MathExprEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/MathExprEditor.tsx new file mode 100644 index 00000000..4c31cdc1 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/MathExprEditor.tsx @@ -0,0 +1,83 @@ +import React, { useState } from "react"; + +import { SelectableValue } from "@grafana/data"; +import { QueryBuilderOperationParamEditorProps, toOption } from "@grafana/plugin-ui"; +import { InlineField, Input, Select } from "@grafana/ui"; + +import { quoteString, getValue } from "../utils/stringHandler"; +import { buildSplitString, SplitString, splitString } from "../utils/stringSplitter"; + +import { getFieldNameOptions } from "./utils/editorHelper"; + +export default function MathExprEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const { expr, resultField } = parseMathExprValue(String(value || "")); + + const updateValue = (expr: string, resultField: string) => { + onChange(index, `${expr} as ${quoteString(resultField)}`); + } + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldNameOptions(props)); + setIsLoading(false); + } + + return ( + <> + { updateValue(e.currentTarget.value, resultField) }} + placeholder="Enter math expression" + /> +
as
+ + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => updateValue(expr, value)} + value={toOption(resultField)} + width="auto" + /> + + + ) +} + +function parseExpr(str: SplitString[]): string { + if (str.length === 0) { + return ""; + } + let token = str[0]; + let i = 0; + while (i < str.length && (token = str[i])) { + if (token.type === "space" && token.value === "as") { + break; + } + i++; + } + return buildSplitString(str.splice(0, i)); +} + +function parseMathExprValue(value: string) { + // expr1 as resultName1 + let str = splitString(value); + let expr = ""; + let resultField = ""; + + if (str.length > 0 && str[0].value !== "as") { + expr = parseExpr(str); + } + if (str.length > 0 && str[0].value === "as") { + str.shift(); + } + if (str.length > 0) { + resultField = getValue(str[0]); + } + return { expr, resultField }; +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/NumberEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/NumberEditor.tsx new file mode 100644 index 00000000..cd6951e3 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/NumberEditor.tsx @@ -0,0 +1,32 @@ +import React from "react"; + +import { QueryBuilderOperationParamEditorProps } from "@grafana/plugin-ui"; + +export default function NumberEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const validateInput = (input: string): string => { + // allow numbers, -, +, ., KB, MB, GB, TB, Ki, Mi, Gi, Ti, K,M, G, T, KiB, MiB, GiB, TiB, _ + const allowedPattern = /[^0-9\-+._KMGTkmgtiIBbnf]/g; + return input.replace(allowedPattern, ''); + } + const handleInputChange = (e: React.ChangeEvent) => { + const inputValue = e.target.value; + const sanitizedValue = validateInput(inputValue); + if (sanitizedValue !== inputValue) { + e.target.value = sanitizedValue; + } + }; + const updateValue = (e: React.FocusEvent) => { + onChange(index, e.target.value.trim()); + } + + return ( + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/QueryEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/QueryEditor.tsx new file mode 100644 index 00000000..002dc704 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/QueryEditor.tsx @@ -0,0 +1,25 @@ +import React from "react"; + +import { QueryBuilderOperationParamEditorProps } from "@grafana/plugin-ui"; +import { InlineField } from "@grafana/ui"; + +import { VictoriaLogsDatasource } from "../../../../datasource"; +import { QueryBuilderContainer } from "../QueryBuilderContainer"; + +export default function QueryEditor(props: QueryBuilderOperationParamEditorProps) { + const { datasource, value, timeRange, onRunQuery, onChange } = props; + const onVisQueryChange = (update: { expr: string }) => { + onChange(props.index, update.expr); + }; + return ( + + + + ) +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/ResultFieldEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/ResultFieldEditor.tsx new file mode 100644 index 00000000..3068f33b --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/ResultFieldEditor.tsx @@ -0,0 +1,32 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps, toOption } from '@grafana/plugin-ui'; +import { Select } from '@grafana/ui'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +export default function ResultFieldEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getFieldNameOptions(props)); + setIsLoading(false); + } + + return ( + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => onChange(index, value)} + value={toOption(String(value || ""))} + width="auto" + /> + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/SingleCharInput.tsx b/src/components/QueryEditor/QueryBuilder/Editors/SingleCharInput.tsx new file mode 100644 index 00000000..25814a9c --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/SingleCharInput.tsx @@ -0,0 +1,36 @@ +import React from "react"; + +import { QueryBuilderOperationParamEditorProps } from "@grafana/plugin-ui"; +import { Input } from "@grafana/ui"; + +export default function SingleCharInput(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const handleChange = (e: React.ChangeEvent) => { + onChange(index, e.target.value); + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Backspace' || e.key === 'Delete') { + (e.target as HTMLInputElement).value = ""; + e.preventDefault(); + return; + } + if (!e.ctrlKey && !e.metaKey && !e.altKey) { + (e.target as HTMLInputElement).value = e.key; + e.preventDefault(); + return; + } + }; + + return ( + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/SortedFieldsEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/SortedFieldsEditor.tsx new file mode 100644 index 00000000..443cc994 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/SortedFieldsEditor.tsx @@ -0,0 +1,125 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { ActionMeta, FormatOptionLabelMeta, Icon, MultiSelect } from '@grafana/ui'; + +import { getValue, quoteString, unquoteString } from '../utils/stringHandler'; +import { splitByUnescapedChar, SplitString, splitString } from '../utils/stringSplitter'; + +import { getFieldNameOptions } from './utils/editorHelper'; + +interface FieldWithDirection { + name: string; + isDesc: boolean; +} + +export default function SortedFieldsEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const str = splitString(String(value || "")); + const parsedValues = parseInputValues(str); + const [values, setValues] = useState(parsedValues); + + const setFields = (newValues: FieldWithDirection[], action?: ActionMeta) => { + if (action) { + if (action.action === "remove-value") { + newValues = values.filter((v) => v.name !== (action.removedValue as any).name); + } + } + setValues(newValues); + const newValue = newValues.map((field) => { + const isRaw = field.isDesc === undefined; + if (isRaw) { + return quoteString(field as unknown as string); + } + return field.isDesc ? `${quoteString(field.name)} desc` : `${quoteString(field.name)}`; + }).join(', '); + onChange(index, newValue); + } + + const toggleDirection = (index: number) => { + const field = values[index]; + field.isDesc = !field.isDesc; + setFields(values); + }; + + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + let options = await getFieldNameOptions(props); + const selectedNames = values.map(v => v.name); + options = options.filter((opt: SelectableValue) => opt.value && !selectedNames.includes(opt.value)); + setOptions(options); + setIsLoading(false); + } + + const handleFormatOptionLabel = (option: SelectableValue, { context }: FormatOptionLabelMeta) => { + if (context === 'value') { + const field = option as FieldWithDirection; + const handleToggle = (e: React.SyntheticEvent) => { + e.stopPropagation(); + const idx = values.findIndex((v) => (v).name === field.name); + if (idx !== -1) { + toggleDirection(idx); + } + }; + return ( + + {formatFieldLabel(field)} + + + ); + } + return <>{option.label}; + } + + return ( + + openMenuOnFocus + onOpenMenu={handleOpenMenu} + isLoading={isLoading} + allowCustomValue + allowCreateWhileLoading + noOptionsMessage="No labels found" + loadingMessage="Loading labels" + options={options} + value={values} + onChange={(values, action) => setFields(values.map((v) => v.value || v as FieldWithDirection), action)} + formatOptionLabel={handleFormatOptionLabel} + /> + ); +} + +const formatFieldLabel = (field: FieldWithDirection): string => { + return field.isDesc ? `${field.name} (desc)` : field.name; +}; + +const parseInputValues = (str: SplitString[]): FieldWithDirection[] => { + let fields: FieldWithDirection[] = []; + for (const field of splitByUnescapedChar(str, ',')) { + if (field.length === 2 && field[1].type === "space" && field[0].type !== "bracket") { + const fieldName = getValue(field[0]); + const isDesc = field[1].value.toLowerCase() === 'desc'; + fields.push({ name: fieldName, isDesc }); + } else if (field.length === 1) { + if (field[0].type === "space") { + fields.push({ name: field[0].value, isDesc: false }); + } else if (field[0].type === "quote") { + const fieldName = unquoteString(field[0].value); + fields.push({ name: fieldName, isDesc: false }); + } + } + } + return fields; +}; diff --git a/src/components/QueryEditor/QueryBuilder/Editors/StatsEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/StatsEditor.tsx new file mode 100644 index 00000000..3680eea3 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/StatsEditor.tsx @@ -0,0 +1,72 @@ +import React, { useMemo } from "react"; + +import { OperationList, QueryBuilderOperation, QueryBuilderOperationParamEditorProps, VisualQueryModeller } from "@grafana/plugin-ui"; + +import { VisualQuery } from "../../../../types"; +import { OperationDefinitions } from "../Operations"; +import { createQueryModellerForCategories } from "../QueryModeller"; +import { QueryModeller } from "../QueryModellerClass"; +import { VictoriaLogsQueryOperationCategory } from "../VictoriaLogsQueryOperationCategory"; +import { parseStatsOperation } from "../utils/operationParser"; +import { splitByUnescapedChar, splitString } from "../utils/stringSplitter"; + +function isObj(v: unknown): v is { expr: string; visQuery: VisualQuery } { + return !!v && typeof v === "object" && "expr" in (v as any) && "visQuery" in (v as any); +} + +export default function StatsEditor(type: "stats" | "running" | "total" = "stats") { + function StatsEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index, datasource, timeRange, onRunQuery, queryModeller } = props; + /* + value: + count() logs_total, count_uniq(ip) ips_total + + count() if (GET) gets, + count() if (POST) posts, + count() if (PUT) puts, + count() total + */ + const visQuery = useMemo(() => isObj(value) ? value.visQuery : parseStatsValue(String(value || ""), queryModeller), [value, queryModeller]); + + const queryStatsModeller = useMemo(() => { + if (type === "stats") { + return createQueryModellerForCategories([VictoriaLogsQueryOperationCategory.Stats]); + } else if (type === "running") { + const operations = new OperationDefinitions().getRunningStatsOperationDefinitions(); + return new QueryModeller(operations); + } + const operations = new OperationDefinitions().getTotalStatsOperationDefinitions(); + return new QueryModeller(operations); + }, []); + const onEditorChange = (query: VisualQuery) => { + const expr = queryStatsModeller.renderOperations("", query.operations); + const next = { expr, visQuery: query }; + onChange(index, next as unknown as string); + } + return ( + + ) + } + return StatsEditor; +} + +function parseStatsValue(value: string, queryModeller: VisualQueryModeller): VisualQuery { + let operations: QueryBuilderOperation[] = []; + if (value !== "") { + let str = splitString(value); + for (const commaPart of splitByUnescapedChar(str, ",")) { + const operation = parseStatsOperation(commaPart, queryModeller); + if (operation) { + operations.push(operation.operation); + } + } + } + return { operations, labels: [], expr: value }; +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.test.tsx b/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.test.tsx new file mode 100644 index 00000000..cd29101d --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.test.tsx @@ -0,0 +1,38 @@ +import { parseStreamFilterValue } from "./StreamFieldEditor"; + +describe("parseStreamFilterValue", () => { + it("should parse a simple label", () => { + const result = parseStreamFilterValue("app"); + expect(result).toEqual({ label: "app", not_in: false, values: [] }); + }); + + it("should parse a label with in operator", () => { + const result = parseStreamFilterValue("app in (nginx, foo.bar)"); + expect(result).toEqual({ label: "app", not_in: false, values: ["nginx", "foo.bar"] }); + }); + + it("should parse a label with not_in operator", () => { + const result = parseStreamFilterValue("app not_in (nginx, foo.bar)"); + expect(result).toEqual({ label: "app", not_in: true, values: ["nginx", "foo.bar"] }); + }); + + it("should parse a label with regex match operator", () => { + const result = parseStreamFilterValue("app=~\"nginx|foo\\.bar\""); + expect(result).toEqual({ label: "app", not_in: false, values: ["nginx", "foo.bar"] }); + }); + + it("should parse a label with regex not match operator", () => { + const result = parseStreamFilterValue("app!~\"nginx|foo\\.bar\""); + expect(result).toEqual({ label: "app", not_in: true, values: ["nginx", "foo.bar"] }); + }); + + it("should handle empty value", () => { + const result = parseStreamFilterValue(""); + expect(result).toEqual({ label: "", not_in: false, values: [] }); + }); + + it("should handle complex cases with escaped characters", () => { + const result = parseStreamFilterValue("label in (v1, v2, v3)"); + expect(result).toEqual({ label: "label", not_in: false, values: ["v1", "v2", "v3"] }); + }); +}); diff --git a/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.tsx new file mode 100644 index 00000000..3ec4b38f --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/StreamFieldEditor.tsx @@ -0,0 +1,203 @@ +import React, { useState } from 'react'; + +import { SelectableValue, toOption } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { Stack, MultiSelect, RadioButtonGroup, InlineField, Select, ActionMeta } from '@grafana/ui'; + +import { FilterFieldType } from "../../../../types"; +import { getValuesFromBrackets } from '../utils/operationParser'; +import { getValue, isValue, quoteString } from '../utils/stringHandler'; +import { splitByUnescapedPipe, SplitString, splitString } from '../utils/stringSplitter'; + +import { getFieldOptions } from './utils/editorHelper'; + +export function parseNonBraketValue(value: SplitString[]): { label: string, not_in: boolean, values: string[] } { + let label = getValue(value[0]); + if (["=", "!", "~"].includes(label)) { + label = ""; + } else { + value.shift(); + } + let is_regex = false; + let not_in = false; + let values: string[] = []; + // get the operator + if (value[0]) { + if (value[0].value === "!") { + not_in = true; + value = value.slice(1); + if (value[0]) { + if (value[0].value === "~") { + is_regex = true; + value.shift(); + } else if (value[0].value === "=") { + value.shift(); + } + } + } else if (value[0].value === "not_in") { + not_in = true; + value.shift(); + } else if (value[0].value === "=") { + value = value.slice(1); + if (value[0] && value[0].value === "~") { + is_regex = true; + value.shift(); + } + } else if (value[0].value === "in") { + value.shift(); + } + } + // get the values + if (value[0]) { + const splitValue = splitString(getValue(value[0])); + if (is_regex) { + for (const group of splitByUnescapedPipe(splitValue)) { + let val = ""; + for (const part of group) { + if (part.type === "colon") { + val += part.value + ":"; + } else if (part.type === "bracket") { + val += part.raw_value; + } else { + val += part.value; + } + } + values.push(val.replace(/\\(.)/g, "$1")); + } + } else if (splitValue.length && isValue(splitValue[0])) { + values = [getValue(splitValue[0])]; + } + } + return { label, not_in, values }; +} + +export function parseStreamFilterValue(value: string): { label: string, not_in: boolean, values: string[] } { + // possible values: {app="nginx"}, {label in (v1,...,vN)}, {label not_in (v1,...,vN)}, {label=~"v1|...|vN"}, {label!~"v1|...|vN"}, {app in ("nginx", "foo.bar")}, {app=~"nginx|foo\\.bar"} + if (!value) { + return { label: "", not_in: false, values: [] }; + } + const splitBracket = splitString(value); + if (!splitBracket.length) { + return { label: "", not_in: false, values: [] }; + } + let label = ""; + if (isValue(splitBracket[0])) { + label = getValue(splitBracket[0]); + } + // Bracket case: {label in (...)} or {label not_in (...)} + const last = splitBracket[splitBracket.length - 1]; + if (last.type === "bracket") { + const not_in = last.prefix === "not_in" || (splitBracket[1] && splitBracket[1].value === "not_in"); + const values = getValuesFromBrackets(last.value); + return { label, not_in, values }; + } + // Non-bracket case: {label=...}, {label=~...}, {label!~...}, etc. + if (isValue(splitBracket[0])) { + return parseNonBraketValue(splitBracket); + } + return { label, not_in: false, values: [] }; +} + +function buildStreamFilterValue(label: string, values: string[], not_in: boolean): string { + const operator = not_in ? "!=" : "="; + values = values.map((value) => quoteString(value)); + if (values.length === 0) { + return `${quoteString(label)}${operator}`; + } else if (values.length === 1) { + return `${quoteString(label)}${operator}${values[0]}`; + } else { + const operator = not_in ? "not_in" : "in"; + return `${quoteString(label)} ${operator} (${values.join(",")})`; + } +} + +export default function StreamFieldEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + + const initialStreamSelector = parseStreamFilterValue(String(value || "")); + const [field, setField] = useState(initialStreamSelector.label); + const [values, setValues] = useState(initialStreamSelector.values); + const [valuesNotIn, setValuesNotIn] = useState(initialStreamSelector.not_in); + const [isLoadingLabelValues, setIsLoadingLabelValues] = useState(false); + const [labelValues, setLabelValues] = useState([]); + + const updateField = async ({ value = "" }) => { + if (field === value) { + return; + } + setField(value); + onChange(index, buildStreamFilterValue(value, [], valuesNotIn)); + }; + + const updateValues = (rawValues: SelectableValue[], action: ActionMeta) => { + let newValues = rawValues.map(({ value = "" }) => value); + if (action) { + if (action.action === "remove-value") { + newValues = values.filter((v) => v !== (action.removedValue as SelectableValue).value); + } + } + setValues(newValues); + onChange(index, buildStreamFilterValue(field, newValues, valuesNotIn)); + }; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenNamesMenu = async () => { + setIsLoading(true); + const options = await getFieldOptions(props, FilterFieldType.StreamFieldNames); + setOptions(options); + setIsLoading(false); + } + + const handleOpenValuesMenu = async () => { + if (field === "") { + return; + } + setIsLoadingLabelValues(true); + const options = await getFieldOptions(props, FilterFieldType.StreamFieldValues, "", field); + setLabelValues(options); + setIsLoadingLabelValues(false); + } + + return ( + + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenNamesMenu} + options={options} + onChange={updateField} + value={toOption(field)} + width="auto" + /> +
+ { + onChange(index, buildStreamFilterValue(field, values, value)); + setValuesNotIn(value); + }} + size="sm" + /> +
+ + + openMenuOnFocus + onOpenMenu={handleOpenValuesMenu} + isLoading={isLoadingLabelValues} + allowCustomValue + allowCreateWhileLoading + loadingMessage="Loading labels" + options={labelValues} + value={values} + onChange={updateValues} + /> + + + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/SubqueryEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/SubqueryEditor.tsx new file mode 100644 index 00000000..dc769f36 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/SubqueryEditor.tsx @@ -0,0 +1,255 @@ +import React, { useMemo, useState } from 'react'; + +import { SelectableValue, toOption } from '@grafana/data'; +import { OperationList, QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { RadioButtonGroup, MultiSelect, Select, ActionMeta } from '@grafana/ui'; + +import { VisualQuery } from '../../../../types'; +import { VictoriaLogsOperationId } from '../Operations'; +import { buildVisualQueryToString, parseExprToVisualQuery } from "../QueryModeller"; +import { getValue, isValue, quoteString, unquoteString } from '../utils/stringHandler'; +import { SplitString, splitString } from '../utils/stringSplitter'; + +import { getFieldNameOptions, getFieldValueOptions } from './utils/editorHelper'; + +export const checkLegacyMultiExact = (str: SplitString[]): boolean => { + // (="error" OR ="fatal") + if (str.length < 2) { + return false; + } + while (str.length >= 2) { + if (!(str[0].type === "space" && str[0].value === "=")) { + return false; + } + if (str[1].type === "bracket") { + return false; + } + str = str.slice(2); + if (!str.length || str[0].type !== "space" || str[0].value.toLowerCase() !== "or") { + return false; + } + str = str.slice(1); + } + return true; +} + +function parseLegacyMultiExact(str: SplitString[]): string[] { + let values: string[] = []; + while (str.length >= 2) { + if (!(str[0].type === "space" && str[0].value === "=")) { + break; + } + if (str[1].type === "bracket") { + break; + } + if (isValue(str[1])) { + values.push(getValue(str[1])); + } + str = str.slice(2); + if (!str.length || str[0].type !== "space" || str[0].value.toLowerCase() !== "or") { + break; + } + str = str.slice(1); + } + return values; +} + +function parseSubquery(value: string): { values: string[]; isQuery: boolean; query: string; fieldName: string } { + let str = splitString(value); + if (str.length === 0) { // empty === stream_id + return { values: [], isQuery: false, query: "", fieldName: "" }; + } else if (str[0].type !== "bracket") { // stream_id + let value = str[0].value; + return { values: [unquoteString(value)], isQuery: false, query: "", fieldName: "" }; + } else if (checkLegacyMultiExact(str[0].value)) { // legacy multi exact + const values = parseLegacyMultiExact(str[0].value); + return { values, isQuery: false, query: "", fieldName: "" }; + } + const query = str[0].raw_value.slice(1, -1); + if (query === "") { + return { values: [], isQuery: false, query: "", fieldName: "" }; + } + const visQuery = parseExprToVisualQuery(query); + const lastOp = visQuery.query.operations[visQuery.query.operations.length - 1]; + const isQuery = lastOp ? [VictoriaLogsOperationId.Fields, VictoriaLogsOperationId.Uniq].includes(lastOp.id as VictoriaLogsOperationId) : false; + if (isQuery) { + const lastOp = visQuery.query.operations[visQuery.query.operations.length - 1]; + const fieldName = unquoteString(lastOp.params[0] as string); + visQuery.query.operations.pop(); + return { values: [], isQuery: true, query: buildVisualQueryToString(visQuery.query), fieldName }; + } else { + let values: string[] = []; + const str = splitString(query); + for (const value of str) { + if (value.type === "space" && value.value === ",") { + continue; + } + if (isValue(value)) { + values.push(getValue(value)); + } + } + return { values, isQuery: false, query: "", fieldName: "" }; + } +} + +function isObj(v: unknown): v is { expr: string; visQuery: VisualQuery } { + return !!v && typeof v === "object" && "expr" in (v as any) && "visQuery" in (v as any); +} + +export default function SubqueryEditor(props: QueryBuilderOperationParamEditorProps) { + const { datasource, timeRange, onRunQuery, onChange, index, value, operation, queryModeller } = props; + const paramLen = operation.params.length; + // paramLen = 1 -> StreamId (single value possible) + // paramLen = 2 -> Multi Exact, contains_all, contains_any + const isStreamIdFilter = paramLen === 1; + let stdFieldName = ""; + if (isStreamIdFilter) { + stdFieldName = "_stream_id"; + } else { + stdFieldName = operation.params[0] as string; + } + const parsedSubquery = useMemo(() => { + if (isObj(value)) { + return parseSubquery(String(value.expr || "")) + } else { + return parseSubquery(String(value || "")) + } + }, [value]); + const { values, isQuery, query: queryValue, fieldName } = parsedSubquery; + + const [filterValues, setFilterValues] = useState(values); + const [useQueryAsValue, setUseQueryAsValue] = useState(isQuery); + const [selectQuery, setSelectQuery] = useState<{ expr: string, visQuery: VisualQuery }>({ + expr: queryValue, + visQuery: isObj(value) ? value.visQuery : parseExprToVisualQuery(queryValue).query + }) + const [queryField, setQueryField] = useState(fieldName); + + const buildSubqueryValue = (newValues: SelectableValue[], action?: ActionMeta) => { + let strValues = newValues.map(v => v.value).filter(Boolean); + if (action) { + if (action.action === "remove-value") { + strValues = values.filter((v) => v !== (action.removedValue as SelectableValue).value); + } + } + setFilterValues(strValues); + const valueExpr = "(" + strValues.map(value => { + if (value.startsWith("$")) { + return value; + } + return quoteString(value); + }).join(", ") + ")"; + if (isStreamIdFilter) { + if (strValues.length === 1) { + onChange(index, strValues[0]); + return; + } + onChange(index, "in" + valueExpr); + return; + } + onChange(index, valueExpr); + } + + const buildSubquery = (visQuery: VisualQuery, fieldName: string, stdFieldName: string) => { + const query = visQuery.expr; + fieldName = fieldName.trim(); + if (fieldName === "") { + fieldName = stdFieldName; + } + let queryExpr = "( "; + if (query !== "") { + queryExpr += query + " | "; + } + queryExpr += `fields ${quoteString(fieldName)})`; + const next = { expr: queryExpr, visQuery }; + onChange(index, next as unknown as string); + } + + const onEditorChange = (query: VisualQuery) => { + query.expr = buildVisualQueryToString(query); + buildSubquery(query, queryField, stdFieldName); + setSelectQuery({ expr: query.expr, visQuery: query }) + }; + const [fieldNames, setFieldNames] = useState[]>([]) + const [isLoadingFieldNames, setIsLoadingFieldNames] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + const handleOpenMenu = async () => { + setIsLoadingFieldNames(true); + const options = await getFieldValueOptions(props, stdFieldName); + setFieldNames(options) + setIsLoadingFieldNames(false) + } + const onQueryValueToggle = (value: boolean) => { + setUseQueryAsValue(value); + if (value) { + // visQuery is empty here + buildSubquery(selectQuery.visQuery, queryField, stdFieldName) + } else { + buildSubqueryValue([]); + } + } + const handleOpenFieldNameMenu = async () => { + setIsLoading(true); + const valueQueryProps = { ...props, query: selectQuery.visQuery }; + setOptions(await getFieldNameOptions(valueQueryProps)); + setIsLoading(false); + } + return ( + <> +
+
+ +
+ {!useQueryAsValue && + + onChange={buildSubqueryValue} + options={fieldNames} + value={filterValues} + isLoading={isLoadingFieldNames} + allowCustomValue + allowCreateWhileLoading + noOptionsMessage="No labels found" + loadingMessage="Loading labels" + width={30} + onOpenMenu={handleOpenMenu} + /> + } + {useQueryAsValue && + <> + + FieldName + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenFieldNameMenu} + options={options} + onChange={({ value = "" }) => { + setQueryField(value); + buildSubquery(selectQuery.visQuery, value, stdFieldName); + }} + value={toOption(queryField)} + width="auto" + /> + + } +
+ + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/UnpackedFieldsSelector.tsx b/src/components/QueryEditor/QueryBuilder/Editors/UnpackedFieldsSelector.tsx new file mode 100644 index 00000000..3753999e --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/UnpackedFieldsSelector.tsx @@ -0,0 +1,88 @@ +import React, { useState } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { QueryBuilderOperationParamEditorProps } from '@grafana/plugin-ui'; +import { ActionMeta, MultiSelect } from '@grafana/ui'; + +import { FilterFieldType, VisualQuery } from "../../../../types"; +import { getValuesFromBrackets } from '../utils/operationParser'; +import { quoteString } from '../utils/stringHandler'; +import { splitString } from '../utils/stringSplitter'; + +export default function UnpackedFieldsSelector(unpackOperation: "unpack_json" | "unpack_logfmt" | "unpack_syslog") { + function UnpackedFieldsSelectorEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index, datasource, timeRange, query, operation, queryModeller } = props; + + const str = splitString(String(value || "")); + const [values, setValues] = useState[]>(toOption(getValuesFromBrackets(str))); + + const setFields = (newValues: SelectableValue[], action: ActionMeta) => { + if (action) { + if (action.action === "remove-value") { + newValues = values.filter((v) => v.value !== (action.removedValue as SelectableValue).value); + } + } + setValues(newValues); + const newValue = newValues + .map(({ value = "" }) => value) + .filter(Boolean) + .map(v => quoteString(v)) + .join(", "); + if (newValues.length === 0) { + onChange(index, ""); + } else { + onChange(index, newValue); + } + } + + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + const fieldName = operation.params[0] as string; + const operations = (query as VisualQuery).operations; + const operationIdx = operations.findIndex(op => op === operation); + const prevOperations = operations.slice(0, operationIdx); + let prevExpr = queryModeller.renderQuery({ operations: prevOperations, labels: [] }); + const unpackedQuery = ` | fields "${fieldName}" | ${unpackOperation} from "${fieldName}" result_prefix "result_" | delete "${fieldName}" `; + if (prevExpr === "") { + prevExpr = "*"; + } + const queryExpr = prevExpr + unpackedQuery; + let options = await datasource.languageProvider?.getFieldList({ query: queryExpr, timeRange, type: FilterFieldType.FieldName }); + options = options ? options.map(({ value, hits }: { value: string; hits: number }) => ({ + value: value.replace(/^(result_)/, ""), + label: value.replace(/^(result_)/, "") || " ", + description: `hits: ${hits}`, + })) : [] + setOptions(options); + setIsLoading(false); + } + + return ( + + openMenuOnFocus + onOpenMenu={handleOpenMenu} + isLoading={isLoading} + allowCustomValue + allowCreateWhileLoading + noOptionsMessage="No labels found" + loadingMessage="Loading labels" + options={options} + value={values} + onChange={setFields} + /> + ); + } + return UnpackedFieldsSelectorEditor; +} + +const toOption = ( + values: string[] +): SelectableValue[] => { + values = values.filter(Boolean); + return values.map((value) => { + return { label: value?.toString(), value }; + }) +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/VariableEditor.tsx b/src/components/QueryEditor/QueryBuilder/Editors/VariableEditor.tsx new file mode 100644 index 00000000..c8652344 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/VariableEditor.tsx @@ -0,0 +1,32 @@ +import React, { useState } from "react"; + +import { SelectableValue } from "@grafana/data"; +import { QueryBuilderOperationParamEditorProps, toOption } from "@grafana/plugin-ui"; +import { Select } from "@grafana/ui"; + +import { getVariableOptions } from "./utils/editorHelper"; + +export default function VariableEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, onChange, index } = props; + const [isLoading, setIsLoading] = useState(false); + const [options, setOptions] = useState[]>([]); + + const handleOpenMenu = async () => { + setIsLoading(true); + setOptions(await getVariableOptions()); + setIsLoading(false); + } + + return ( + + allowCustomValue={true} + allowCreateWhileLoading={true} + isLoading={isLoading} + onOpenMenu={handleOpenMenu} + options={options} + onChange={({ value = "" }) => onChange(index, value)} + value={toOption(String(value || ""))} + width="auto" + /> + ); +} diff --git a/src/components/QueryEditor/QueryBuilder/Editors/utils/editorHelper.ts b/src/components/QueryEditor/QueryBuilder/Editors/utils/editorHelper.ts new file mode 100644 index 00000000..21ae2c2f --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Editors/utils/editorHelper.ts @@ -0,0 +1,113 @@ +import { QueryBuilderOperation, QueryBuilderOperationParamEditorProps, VisualQueryModeller } from "@grafana/plugin-ui"; +import { getTemplateSrv } from "@grafana/runtime"; + +import { VictoriaLogsDatasource } from "../../../../../datasource"; +import { FilterFieldType, VisualQuery } from "../../../../../types"; +import { buildVisualQueryToString } from "../../QueryModeller"; +import { VictoriaLogsQueryOperationCategory } from "../../VictoriaLogsQueryOperationCategory"; + +export async function getVariableOptions() { + return getTemplateSrv().getVariables().map((v: any) => ({ + value: "$" + v.name, + label: "$" + v.name, + description: 'variable', + })); +} + +function startsWithFilterOperation(operations: QueryBuilderOperation[], queryModeller: VisualQueryModeller) { + if (operations.length === 0) { + return false; + } + const firstOp = operations[0]; + const operation = queryModeller.getOperationDefinition(firstOp.id); + if (!operation) { + return false; + } + if ([VictoriaLogsQueryOperationCategory.Filters, VictoriaLogsQueryOperationCategory.Operators, VictoriaLogsQueryOperationCategory.Special].includes(operation.category as VictoriaLogsQueryOperationCategory)) { + return true; + } + return false +} + +export async function getFieldNameOptions(props: QueryBuilderOperationParamEditorProps) { + const { datasource, timeRange, queryModeller, query, operation } = props; + const operations = (query as VisualQuery).operations; + const operationIdx = operations.findIndex(op => op === operation); + const prevOperations = operations.slice(0, (operationIdx === -1) ? operations.length : operationIdx); + const prevExpr = buildVisualQueryToString({ operations: prevOperations, labels: [], expr: "" }); + let expr = ""; + if (prevExpr.trim() !== "" && prevExpr.trim() !== "\"\"") { + const firstOpIsFilter = startsWithFilterOperation(operations, queryModeller); + if (!firstOpIsFilter) { + expr = "_msg:* | "; + } + expr += prevExpr; + } else { + expr = "_msg:*"; + } + const replacedExpr = (datasource as VictoriaLogsDatasource).interpolateString(expr); + let options = []; + try { + options = await datasource.languageProvider?.getFieldList({ query: replacedExpr, timeRange, type: FilterFieldType.FieldName }); + } catch (e) { + console.warn("Error fetching field names", e, "query", replacedExpr); + options = await datasource.languageProvider?.getFieldList({ timeRange, type: FilterFieldType.FieldName }); + } + options = options.map(({ value, hits }: { value: string; hits: number }) => ({ + value, + label: value || " ", + description: `hits: ${hits}`, + })); + return [...options, ...await getVariableOptions()]; +} + +export async function getFieldOptions(props: QueryBuilderOperationParamEditorProps, fieldType: FilterFieldType, suffixQuery = "", fieldName?: string) { + if (fieldName === undefined || fieldName.trim() === "") { + if (fieldType === FilterFieldType.FieldValue || fieldType === FilterFieldType.StreamFieldValues) { + throw new Error("fieldName is required for FieldValue and StreamFieldValues fieldType"); + } + } + const { datasource, timeRange, query, operation, queryModeller } = props; + const operations = (query as VisualQuery).operations; + const operationIdx = operations.findIndex(op => op === operation); + const prevOperations = operations.slice(0, (operationIdx === -1) ? operations.length : operationIdx); + const prevExpr = buildVisualQueryToString({ operations: prevOperations, labels: [], expr: "" }); + let expr; + if (prevExpr.trim() !== "" && prevExpr.trim() !== "\"\"") { + const firstOpIsFilter = startsWithFilterOperation(operations, queryModeller); + if (!firstOpIsFilter) { + expr = "_msg:* | "; + } + expr = prevExpr; + } else { + expr = "_msg:*"; + } + if (suffixQuery.trim() !== "") { + expr += " | " + suffixQuery; + } + const replacedExpr = (datasource as VictoriaLogsDatasource).interpolateString(expr); + let options = []; + try { + options = await datasource.languageProvider?.getFieldList({ query: replacedExpr, timeRange, type: fieldType, field: fieldName }); + } catch (e) { + console.warn("Error fetching field names", e, "query", replacedExpr); + options = await datasource.languageProvider?.getFieldList({ timeRange, type: fieldType, field: fieldName }); + } + options = options.map(({ value, hits }: { value: string; hits: number }) => ({ + value, + label: value || " ", + description: `hits: ${hits}`, + })); + return [...options, ...await getVariableOptions()]; +} + +export async function getFieldValueOptions(props: QueryBuilderOperationParamEditorProps, fieldName: string, suffixQuery = "") { + return getFieldOptions(props, FilterFieldType.FieldValue, suffixQuery, fieldName); +} + +export async function getValueTypeOptions(props: QueryBuilderOperationParamEditorProps) { + const { operation } = props; + const fieldName = operation.params[0] as string; + const valueTypeOperations = `uniq by "${fieldName}" | block_stats`; + return getFieldValueOptions(props, "type", valueTypeOperations); +} diff --git a/src/components/QueryEditor/QueryBuilder/Operations.tsx b/src/components/QueryEditor/QueryBuilder/Operations.tsx new file mode 100644 index 00000000..838166d7 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/Operations.tsx @@ -0,0 +1,4929 @@ +// LogsQL version v1.34.0 +import React, { FunctionComponent, useMemo, useState } from 'react'; + +import { DataSourceApi } from '@grafana/data'; +import { QueryBuilderOperationDefinition, VisualQueryModeller, QueryBuilderOperationParamEditorProps, QueryBuilderOperationParamValue, QueryBuilderOperation, OperationList } from '@grafana/plugin-ui'; + +import { VisualQuery } from "../../../types"; + +import ExactValueEditor from './Editors/ExactValueEditor'; +import FieldAsFieldEditor from './Editors/FieldAsFieldEditor'; +import FieldEditor from './Editors/FieldEditor'; +import FieldValueTypeEditor from './Editors/FieldValueTypeEditor'; +import FieldsEditor from './Editors/FieldsEditor'; +import FieldsEditorWithPrefix from './Editors/FieldsEditorWithPrefix'; +import LogicalFilterEditor from './Editors/LogicalFilterEditor'; +import MathExprEditor from './Editors/MathExprEditor'; +import NumberEditor from './Editors/NumberEditor'; +import QueryEditor from './Editors/QueryEditor'; +import ResultFieldEditor from './Editors/ResultFieldEditor'; +import SingleCharInput from './Editors/SingleCharInput'; +import SortedFieldsEditor from './Editors/SortedFieldsEditor'; +import StatsEditor from './Editors/StatsEditor'; +import StreamFieldEditor from './Editors/StreamFieldEditor'; +import SubqueryEditor from './Editors/SubqueryEditor'; +import UnpackedFieldsSelector from './Editors/UnpackedFieldsSelector'; +import VariableEditor from './Editors/VariableEditor'; +import { parseExprToVisualQuery } from './QueryModeller'; +import { QueryModeller } from "./QueryModellerClass"; +import { VictoriaLogsQueryOperationCategory } from "./VictoriaLogsQueryOperationCategory"; +import { getValuesFromBrackets, getConditionFromString } from './utils/operationParser'; +import { getValue, isValue, quoteString, unquoteString } from './utils/stringHandler'; +import { buildSplitString, splitByUnescapedChar, SplitString } from './utils/stringSplitter'; + +export enum VictoriaLogsOperationId { + Word = 'word', + Time = 'time', + DayRange = 'day_range', + WeekRange = 'week_range', + Stream = 'stream', + StreamId = 'stream_id', + PatternMatch = 'pattern_match', + Substring = 'substring', + RangeComparison = 'range_comparison', + Exact = 'exact', + MultiExact = 'multi_exact', + ContainsAll = 'contains_all', + ContainsAny = 'contains_any', + EqualsCommonCase = 'equals_common_case', + ContainsCommonCase = 'contains_common_case', + Sequence = 'seq', + Regexp = 'regexp', + Range = 'range', + IPv4Range = 'ipv4_range', + StringRange = 'string_range', + LengthRange = 'len_range', + ValueType = 'value_type', + EqField = 'eq_field', + LeField = 'le_field', + LtField = 'lt_field', + Logical = 'logical', + // Operators + AND = 'and', + OR = 'or', + NOT = 'not', + // Pipes + BlockStats = 'block_stats', + BlocksCount = 'blocks_count', + CollapseNums = 'collapse_nums', + Copy = 'copy', + Decolorize = 'decolorize', + Delete = 'delete', + DropEmptyFields = 'drop_empty_fields', + Extract = 'extract', + ExtractRegexp = 'extract_regexp', + Facets = 'facets', + FieldNmes = 'field_names', + FieldValues = 'field_values', + Fields = 'fields', + First = 'first', + Format = 'format', + GenerateSequence = 'generate_sequence', + Join = 'join', + JsonArrayLen = 'json_array_len', + Hash = 'hash', + Last = 'last', + Len = 'len', + Limit = 'limit', + Math = 'math', + Offset = 'offset', + PackJSON = 'pack_json', + PackLogfmt = 'pack_logfmt', + QueryStats = 'query_stats', + Rename = 'rename', + Replace = 'replace', + ReplaceRegexp = 'replace_regexp', + RunningStats = 'running_stats', + Sample = 'sample', + SetStreamFields = "set_stream_fields", + Sort = 'sort', + Split = 'split', + Stats = 'stats', + StreamContext = 'stream_context', + TimeAdd = 'time_add', + Top = 'top', + TotalStats = 'total_stats', + Union = 'union', + Uniq = 'uniq', + UnpackJson = 'unpack_json', + UnpackLogfmt = 'unpack_logfmt', + UnpackSyslog = 'unpack_syslog', + UnpackWords = 'unpack_words', + Unroll = 'unroll', + // Stats + Avg = 'avg', + Count = 'count', + CountEmpty = 'count_empty', + CountUniq = 'count_uniq', + CountUniqHash = 'count_uniq_hash', + Histogram = 'histogram', + JsonValues = 'json_values', + Max = 'max', + Median = 'median', + Min = 'min', + Quantile = 'quantile', + Rate = 'rate', + RateSum = 'rate_sum', + RowAny = 'row_any', + RowMax = 'row_max', + RowMin = 'row_min', + Sum = 'sum', + SumLen = 'sum_len', + UniqValues = 'uniq_values', + Values = 'values', + // Special + Options = 'options', + FieldContainsAnyValueFromVariable = 'contains_any_from_variable', // multi variable not compatible with normal contains_any + Comment = 'comment', +} + +export interface VictoriaQueryBuilderOperationDefinition extends QueryBuilderOperationDefinition { + /** returns an array of parameter values matching the exact types and order in defaultParams */ + splitStringByParams: (str: SplitString[], fieldName?: string) => { params: QueryBuilderOperationParamValue[], length: number }; +} + +function addVictoriaOperation( + def: QueryBuilderOperationDefinition, + query: VisualQuery, + modeller: VisualQueryModeller +): VisualQuery { + query.operations.push({ + id: def.id, + params: def.defaultParams, + }); + return query; +} + +function parseFieldMapList(str: SplitString[]): { params: QueryBuilderOperationParamValue[], length: number } { + let length = str.length; + let params: string[] = []; + let fromField = "" + let toField = ""; + while (str.length > 0) { + if (str[0].value === ",") { + params.push(""); + str.shift(); + if (str.length === 0) { + params.push(""); + } + continue; + } + if (!isValue(str[0])) { + break; + } + fromField = getValue(str[0]); + str.shift(); + if (str.length === 0 || !(str[0].type === "space" && str[0].value === "as")) { + const quotedFromString = fromField === "" ? '""' : quoteString(fromField); + params.push(`${quotedFromString} as ""`); + break; + } + str = str.slice(1); + if (str.length > 0 && isValue(str[0])) { + toField = getValue(str[0]); + str.shift(); + } + const quotedFromString = fromField === "" ? '""' : quoteString(fromField); + const quotedToString = toField === "" ? '""' : quoteString(toField); + toField = ""; + params.push(`${quotedFromString} as ${quotedToString}`); + } + if (params.length === 0) { + params = [""]; + } + return { params, length: length - str.length }; +} + +function pipeExpr(innerExpr: string, expr: string): string { + return innerExpr === "" ? expr : innerExpr + " | " + expr; +} + +export class OperationDefinitions { + defaultField: string; + operationDefinitions: VictoriaQueryBuilderOperationDefinition[]; + conditionalEditor: FunctionComponent; + constructor(defaultField = "_msg") { + this.defaultField = defaultField; + this.conditionalEditor = this.getConditionalEditor(); + this.operationDefinitions = this.getOperationDefinitions(); + } + + all(): VictoriaQueryBuilderOperationDefinition[] { + return this.operationDefinitions; + } + + getTotalStatsOperationDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + const list: VictoriaQueryBuilderOperationDefinition[] = []; + for (const op of this.operationDefinitions) { + if ([ + VictoriaLogsOperationId.Count, + VictoriaLogsOperationId.Max, + VictoriaLogsOperationId.Min, + VictoriaLogsOperationId.Sum + ].includes(op.id as VictoriaLogsOperationId)) { + list.push(op); + } + } + return list; + } + + getRunningStatsOperationDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return this.getTotalStatsOperationDefinitions(); + } + + getConditionalEditor() { + let filterDefinitions = this.getFilterDefinitions(); + const queryModeller = new QueryModeller(filterDefinitions); + + function ConditionalEditor(props: QueryBuilderOperationParamEditorProps) { + const { value, index, onChange, onRunQuery, datasource, timeRange } = props; + const visQuery = useMemo(() => { + return parseExprToVisualQuery(String(value || "")).query; + }, [value]); + const [state, setState] = useState<{ expr: string, visQuery: VisualQuery }>({ + expr: String(value || ""), + visQuery: visQuery, + }) + const onEditorChange = (query: VisualQuery) => { + const expr = queryModeller.renderQuery(query as VisualQuery); + setState({ expr, visQuery: query }) + onChange(index, expr); + }; + return ( + + ); + } + return ConditionalEditor; + } + + + getPipeDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return [ + { + id: VictoriaLogsOperationId.BlockStats, + name: 'Block stats', + params: [], + defaultParams: [], + toggleable: true, + alternativesKey: "debug", + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => pipeExpr(innerExpr, "block_stats"), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.BlocksCount, + name: 'Blocks count', + params: [], + defaultParams: [], + toggleable: true, + alternativesKey: "debug", + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => pipeExpr(innerExpr, "blocks_count"), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.CollapseNums, + name: 'Collapse nums', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Prettify", + type: "boolean", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "style", + defaultParams: [this.defaultField, false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const prettify = model.params[1] as boolean; + const condition = model.params[2] as string; + let expr = "collapse_nums"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (field !== this.defaultField) { + expr += ` at ${quoteString(field)}`; + } + if (prettify) { + expr += " prettify"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [field: string, prettify: boolean, condition: string] = [this.defaultField, false, ""]; + params[2] = getConditionFromString(str); + if (str.length > 1) { // at _msg + if (str[0].type === "space" && str[0].value === "at") { + str = str.slice(1); + if (isValue(str[0])) { + params[0] = getValue(str[0]); + } + str.shift(); + } + } + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "prettify") { + params[1] = true; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Copy, + name: 'Copy', + params: [{ + name: "Fields", + type: "string", + restParam: true, + editor: FieldAsFieldEditor, + }], + alternativesKey: "change", + defaultParams: ['"" as ""'], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params.filter((v) => Boolean(v) || v === ""); + const expr = "copy " + fields.join(', '); + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseFieldMapList, + }, + { + id: VictoriaLogsOperationId.Decolorize, + name: 'Decolorize', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }], + alternativesKey: "style", + defaultParams: ["_msg"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + let expr = "decolorize"; + if (field !== this.defaultField) { + expr += ` ${quoteString(field)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string] = ["_msg"]; + if (str.length > 0) { + if (isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Delete, + name: 'Delete', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }], + alternativesKey: "change", + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const expr = "delete " + fields; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + const values = parsePrefixFieldList(str); + return { params: [values.join(", ")], length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Fields, + name: 'Fields', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }], + alternativesKey: "reduce", + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const expr = "fields " + fields; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + const length = str.length; + const fields = parsePrefixFieldList(str); + if (fields.length === 0) { + fields.push(""); + } + const param = fields.join(", "); + return { params: [param], length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.DropEmptyFields, + name: 'Drop empty fields', + params: [], + alternativesKey: "change", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + return pipeExpr(innerExpr, "drop_empty_fields"); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.Extract, + name: 'Extract', + params: [{ + name: "From field", + type: "string", + editor: FieldEditor, + }, { + name: "Pattern", + type: "string", + }, { + name: "Keep original fields", + type: "boolean", + optional: true, + }, { + name: "Skip emptry results", + type: "boolean", + optional: true, + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "extract", + defaultParams: [this.defaultField, "", false, false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + explainHandler: () => ` +Pattern: \`text1text2...textNtextN+1\` +
+Where text1, … textN+1 is arbitrary non-empty text, which matches as is to the input text. Anonymous placeholders are written as <_>.`, + renderer: (model, def, innerExpr) => buildExtractOperation(model, innerExpr, this.defaultField), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => parseExtractOperation(str, this.defaultField), + }, + { + id: VictoriaLogsOperationId.ExtractRegexp, + name: 'Extract regex', + params: [{ + name: "From field", + type: "string", + editor: FieldEditor, + }, { + name: "Pattern", + type: "string", + }, { + name: "Keep original fields", + type: "boolean", + }, { + name: "Skip empty results", + type: "boolean", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "extract", + defaultParams: [this.defaultField, "", false, false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => buildExtractOperation(model, innerExpr, this.defaultField), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => parseExtractOperation(str, this.defaultField), + }, + { + id: VictoriaLogsOperationId.Facets, + name: 'Facets', + params: [{ + name: "Number of facets", + type: "number", + optional: true, + }, { + name: "Max Values per Field", + type: "number", + optional: true, + }, { + name: "Max Value Length", + type: "number", + optional: true, + }, { + name: "Keep const fields", + type: "boolean", + optional: true, + }], + alternativesKey: "facets", + defaultParams: [0, 0, 0, false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const [numFacets, maxValuesPerField, maxValueLen, keepConstFields] = model.params as [number, number, number, boolean]; + let expr = "facets "; + if (numFacets > 0) { + expr += numFacets; + } + if (maxValuesPerField > 0) { + expr += " max_values_per_field " + maxValuesPerField; + } + if (maxValueLen > 0) { + expr += " max_value_len " + maxValueLen; + } + if (keepConstFields) { + expr += " keep_const_fields"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [number, number, number, boolean] = [0, 0, 0, false]; + if (str.length > 0) { + const number = parseNumber(str[0], undefined); + if (number !== undefined) { + params[0] = number; + str.shift(); + } + for (let i = 0; i < 3; i++) { + if (str.length > 0 && str[0].type === "space") { // next max_values_per_field/max_value_len/keep_const_fields + if (str[0].value === "keep_const_fields") { + params[3] = true; + str.shift(); + } else if (str[0].value === "max_values_per_field") { + str = str.slice(1); + params[1] = parseNumber(str[0], 0); + str.shift(); + } else if (str[0].value === "max_value_len") { + str = str.slice(1); + params[2] = parseNumber(str[0], 50); + str.shift(); + } + } + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.FieldNmes, + name: 'Field names', + params: [], + alternativesKey: "reduce", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + return pipeExpr(innerExpr, "field_names"); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.FieldValues, + name: 'Field values', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Limit", + type: "number", + optional: true, + }], + alternativesKey: "reduce", + defaultParams: [this.defaultField, 0], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const limit = model.params[1] as number; + let expr = `field_values ${quoteString(field)}`; + if (limit > 0) { + expr += " limit " + limit; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, number] = [this.defaultField, 0]; + if (str.length > 0) { + if (!isValue(str[0])) { + return { params, length: length - str.length }; + } + params[0] = getValue(str[0]); + str.shift(); + if (str.length >= 2) { // limit 10 + if (str[0].type === "space" && str[0].value === "limit") { + str = str.slice(1); + params[1] = parseNumber(str[0], 0); + str.shift(); + } + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.First, + name: 'First', + params: [{ + name: "Number of rows", + type: "number", + }, { + name: "Field", + type: "string", + editor: SortedFieldsEditor, + }, { + name: "Descending", + type: "boolean", + }, { + name: "Parttition by", + type: "string", + editor: FieldsEditor, + }], + alternativesKey: "reduce", + defaultParams: [3, "", false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const numRows = model.params[0] as number; + const field = model.params[1] as string; + const desc = model.params[2] as boolean; + const partitionBy = model.params[3] as string; + let expr = `first ${numRows} by (${field})`; + if (desc) { + expr += " desc"; + } + if (partitionBy !== "") { + expr += ` partition by (${partitionBy})`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseFirstLastPipe, + explainHandler: () => "https://docs.victoriametrics.com/victorialogs/logsql/#first-pipe", + }, + { + id: VictoriaLogsOperationId.Format, + name: 'Format', + params: [{ + name: "Format", + type: "string", + }, { + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "format", + defaultParams: ["", this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const format = model.params[0] as string; + const field = model.params[1] as string; + const condition = model.params[2] as string; + let expr = "format " + if (condition !== "") { + expr += `if (${condition}) `; + } + expr += `'${format}'`; + if (field !== this.defaultField) { + expr += ` as ${quoteString(field)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let params: string[] = ["", this.defaultField, ""]; + let length = str.length; + params[2] = getConditionFromString(str); + if (str.length > 0) { + if (str[0].type !== "quote") { + return { params, length: length - str.length }; + } + params[0] = unquoteString(str[0].value); + str = str.slice(1); + if (str.length > 1) { + if (str[0].type === "space" && str[0].value === "as") { + str = str.slice(1); + if (!isValue(str[0])) { + return { params, length: length - str.length }; + } + params[1] = getValue(str[0]); + str.shift(); + } + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.GenerateSequence, + name: 'Generate sequence', + params: [{ + name: "Number of rows", + type: "number", + }], + defaultParams: [10], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const numRows = model.params[0] as number; + let expr = `generate_sequence ${numRows}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [number] = [10]; + if (str.length > 0) { + const number = parseNumber(str[0], undefined); + if (number !== undefined) { + params[0] = number; + str.shift(); + } + } + return { params, length: length - str.length }; + } + }, + { + id: VictoriaLogsOperationId.Join, + name: 'Join', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Subquery", + type: "string", + editor: QueryEditor, + }, { + name: "Prefix", + type: "string", + }, { + name: "Inner Join", + type: "boolean", + optional: true, + }], + alternativesKey: "combine", + defaultParams: ["", "", "", false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const subquery = model.params[1] as string; + const prefix = model.params[2] as string; + const innerJoin = model.params[3] as boolean; + let expr = `join by (${fields}) (${subquery})`; + if (innerJoin) { + expr += " inner"; + } + if (prefix !== "") { + expr += ` prefix ${quoteString(prefix)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let result: [string, string, string, boolean] = ["", "", "", false]; + do { + if (str.length >= 3) { // join by (user) ( _time:1d {app="app2"} | stats by (user) count() app2_hits ) inner + if (str[0].type === "space" && str[0].value === "by") { + } else { + break; + } + if (str[1].type === "bracket") { + result[0] = str[1].raw_value.slice(1, -1); + } else { + break; + } + if (str[2].type === "bracket") { + result[1] = str[2].raw_value.slice(1, -1); + } else { + break; + } + str = str.slice(3); + for (let i = 0; i < 2; i++) { + if (str.length >= 2) { // prefix "app2." + if (str[0].type === "space" && str[0].value === "prefix") { + if (isValue(str[1])) { + result[2] = getValue(str[1]); + str = str.slice(2); + } + } + } + if (str.length > 0) { // inner + if (str[0].type === "space" && str[0].value === "inner") { + result[3] = true; + str.shift(); + } + } + if (result[3] && result[2] !== "") { + break; + } + } + } + } while (false); + return { params: result, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.JsonArrayLen, + name: 'Json array len', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "count", + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const resultField = model.params[1] as string; + let expr = `json_array_len(${quoteString(field)}) as ${quoteString(resultField)}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => parseLenPipe(str, "json_array_len"), + }, + { + id: VictoriaLogsOperationId.Hash, + name: 'Hash', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "hash", + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const resultField = model.params[1] as string; + let expr = `hash(${quoteString(field)}) as ${quoteString(resultField)}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => parseLenPipe(str, "hash"), + }, + { + id: VictoriaLogsOperationId.Last, + name: 'Last', + params: [{ + name: "Number of rows", + type: "number", + }, { + name: "Field", + type: "string", + editor: SortedFieldsEditor, + }, { + name: "Descending", + type: "boolean", + }, { + name: "Parttition by", + type: "string", + editor: FieldsEditor, + }], + alternativesKey: "reduce", + defaultParams: [0, "", false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const numRows = model.params[0] as number; + const field = model.params[1] as string; + const desc = model.params[2] as boolean; + const partitionBy = model.params[3] as string; + let expr = `last ${numRows} by (${quoteString(field)})`; + if (desc) { + expr += " desc"; + } + if (partitionBy !== "") { + expr += ` partition by (${partitionBy})`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseFirstLastPipe, + }, + { + id: VictoriaLogsOperationId.Len, + name: 'Len', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "count", + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const resultField = model.params[1] as string; + let expr = `len(${quoteString(field)}) as ${quoteString(resultField)}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => parseLenPipe(str, "len"), + }, + { + id: VictoriaLogsOperationId.Limit, + name: 'Limit', + params: [{ + name: "Number of rows", + type: "number", + }], + alternativesKey: "reduce", + defaultParams: [10], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const number = model.params[0] as number; + let expr = "limit " + number; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [parseNumber(str[0], 10)], length: 1 }; + }, + }, + { + id: VictoriaLogsOperationId.Math, + name: 'Math', + params: [{ + name: "", + type: "string", + restParam: true, + editor: MathExprEditor, + }], + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const expr = "math " + model.params.join(', '); + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let expressions: string[] = []; + let expr = ""; + let field = ""; + let exprFinised = false; + while (str.length > 0) { + if (!exprFinised) { + if (str[0].type === "quote") { + expr += str[0].value; + str = str.slice(1); + exprFinised = true; + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "as") { + str.shift(); + if (str.length === 0) { + expressions.push(`${expr} as `); + } + } + } + } else if (str[0].type === "space" && str[0].value === "as") { + str.shift(); + exprFinised = true; + if (str.length === 0) { + expressions.push(`${expr} as `); + } + } else if (str[0].type === "space" && str[0].value === ",") { + exprFinised = true; + } else if (str[0].type === "space") { + expr += str[0].value + " "; + str.shift(); + } else if (str[0].type === "bracket") { + expr += str[0].raw_value; + str.shift(); + } + } else { + if (str[0].value !== ",") { + if (!isValue(str[0])) { + break; + } + field = getValue(str[0]); + str.shift(); + } + expressions.push(`${expr} as ${quoteString(field)}`); + expr = ""; + field = ""; + exprFinised = false; + if (str.length > 0 && str[0].type === "space" && str[0].value === ",") { + str.shift(); + if (str.length === 0) { + expressions.push(""); + } + } else { + break; + } + } + } + if (expressions.length === 0) { + expressions.push(""); + } + return { params: expressions, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Offset, + name: 'Offset', + params: [{ + name: "Row offset", + type: "number", + }], + defaultParams: [0], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const offset = model.params[0] as number; + let expr = "offset " + offset; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [parseNumber(str[0], 10)], length: 1 }; + }, + }, + { + id: VictoriaLogsOperationId.PackJSON, + name: 'Pack JSON', + params: [{ + name: "Source fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Destination field", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "pack", + defaultParams: ["", this.defaultField], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const sourceFields = model.params[0] as string; + const destField = model.params[1] as string; + let expr = "pack_json"; + if (sourceFields !== "") { + expr += ` fields (${sourceFields})`; + } + expr += ` as ${quoteString(destField)}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: this.parsePackPipe, + }, + { + id: VictoriaLogsOperationId.PackLogfmt, + name: 'Pack logfmt', + params: [{ + name: "Source fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Destination field", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "pack", + defaultParams: ["", this.defaultField], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const sourceFields = model.params[0] as string; + const destField = model.params[1] as string; + let expr = "pack_logfmt"; + if (sourceFields !== "") { + expr += ` fields (${sourceFields})`; + } + expr += ` as ${quoteString(destField)}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: this.parsePackPipe, + }, { + id: VictoriaLogsOperationId.QueryStats, + name: 'Query stats', + params: [], + alternativesKey: "debug", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + return pipeExpr(innerExpr, "query_stats"); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + } + }, + { + id: VictoriaLogsOperationId.Rename, + name: 'Rename', + params: [{ + name: "Field", + type: "string", + restParam: true, + editor: FieldAsFieldEditor, + }], + alternativesKey: "change", + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params.filter((v) => Boolean(v) || v === ""); + if (fields.length === 0) { + return innerExpr; + } + const expr = 'rename ' + fields.join(', ') + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseFieldMapList, + }, + { + id: VictoriaLogsOperationId.Replace, + name: 'Replace', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Old value", + type: "string", + }, { + name: "New value", + type: "string", + }, { + name: "Limit", + type: "number", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "replace", + defaultParams: ["", "", "", 0, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const oldValue = model.params[1] as string; + const newValue = model.params[2] as string; + const limit = model.params[3] as number; + const condition = model.params[4] as string; + let expr = "replace"; + if (condition !== "") { + expr += ` if (${condition})`; + } + expr += ` (${quoteString(oldValue, false)}, ${quoteString(newValue, false)})`; + if (field !== this.defaultField) { + expr += ` at ${quoteString(field)}`; + } + if (limit > 0) { + expr += " limit " + limit; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, string, string, number, string] = ["", "", "", 0, ""]; + params[4] = getConditionFromString(str); + do { + if (str.length > 0) { // replace ("secret-password", "***") at _msg + if (str[0].type !== "bracket") { + break; + } + const values = str[0].value; + if (values.length < 2) { + break; + } + if (isValue(values[0])) { + params[1] = getValue(values[0]); + } + if (values.length === 2 && values[1].value === ",") { + params[2] = ""; + } else if (isValue(values[2])) { + params[2] = getValue(values[2]); + } + str = str.slice(1); + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "at") { + str = str.slice(1); + if (str.length > 0 && isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } + } + } + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "limit") { + str = str.slice(1); + if (isValue(str[0])) { + params[3] = parseInt(getValue(str[0]), 10); + str.shift(); + } + } + } + } + } while (false); + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.ReplaceRegexp, + name: 'Replace regex', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Regexp", + type: "string", + }, { + name: "Replacement", + type: "string", + }, { + name: "Limit", + type: "number", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "replace", + defaultParams: ["", "", "", 0, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const oldValue = model.params[1] as string; + const newValue = model.params[2] as string; + const limit = model.params[3] as number; + const condition = model.params[4] as string; + let expr = "replace_regexp"; + if (condition !== "") { + expr += ` if (${condition})`; + } + expr += ` (${quoteString(oldValue, false)}, ${quoteString(newValue, false)})`; + if (field !== this.defaultField) { + expr += ` at ${quoteString(field)}`; + } + if (limit > 0) { + expr += " limit " + limit; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, string, string, number, string] = ["", "", "", 0, ""]; + params[4] = getConditionFromString(str); + do { + if (str.length > 0) { // replace_regexp ("host-(.+?)-foo", "$1") at _msg + if (str[0].type !== "bracket") { + break; + } + const values = str[0].value; + if (values.length < 2) { + break; + } + if (values.length > 0 && isValue(values[0])) { + params[1] = getValue(values[0]); + } + if (values.length === 2 && values[1].value === ",") { + params[2] = ""; + } else if (values.length > 2 && isValue(values[2])) { + params[2] = getValue(values[2]); + } + str = str.slice(1); + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "at") { + str = str.slice(1); + if (str.length > 0 && isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } + } + } + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "limit") { + str = str.slice(1); + if (isValue(str[0])) { + params[3] = parseInt(getValue(str[0]), 10); + str.shift(); + } + } + } + } + } while (false); + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.RunningStats, + name: 'Running stats', + params: [{ + name: "Stats by", + type: "string", + }, { + name: "Stats", + type: "string", + editor: StatsEditor("running"), + }], + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: renderStatsPipe, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsPipe, + }, + { + id: VictoriaLogsOperationId.Sample, + name: 'Sample', + params: [{ + name: "1/N of Samples", + type: "number", + }], + alternativesKey: "reduce", + defaultParams: [1], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const sample = model.params[0] as number; + let expr = "sample " + sample; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [parseNumber(str[0], 1)], length: 1 }; + }, + }, + { + id: VictoriaLogsOperationId.SetStreamFields, + name: 'Set stream fields', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const condition = model.params[1] as string; + let expr = "set_stream_fields"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (fields !== "") { + expr += ` ${fields}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + const length = str.length; + const params: [string, string] = ["", ""]; + params[1] = getConditionFromString(str); + params[0] = buildSplitString(str); + return { params, length: length }; + }, + }, + { + id: VictoriaLogsOperationId.Sort, + name: 'Sort by', + params: [{ + name: "Fields", + type: "string", + editor: SortedFieldsEditor, + }, { + name: "Descending", + type: "boolean", + }, { + name: "Limit", + type: "number", + }, { + name: "Offset", + type: "number", + }, { + name: "Partition by", + type: "string", + editor: FieldsEditor, + }], + defaultParams: ["", false, 0, 0, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const descending = model.params[1] as boolean; + const limit = model.params[2] as number; + const offset = model.params[3] as number; + const partitionBy = model.params[4] as string; + let expr = "sort by (" + fields + ")"; + if (descending) { + expr += " desc"; + } + if (partitionBy !== "") { + expr += ` partition by (${partitionBy})`; + } + if (limit > 0) { + expr += " limit " + limit; + } + if (offset > 0) { + expr += " offset " + offset; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { // sort by (foo, bar) desc + let length = str.length; + let result: [string, boolean, number, number, string] = ["", false, 0, 0, ""]; + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "by") { + str.shift(); + } + if (str[0].type === "bracket") { + result[0] = str[0].raw_value.slice(1, -1); + } else { + return { params: result, length: length - str.length }; + } + str = str.slice(1); + while (str.length > 0) { + if (str.length > 1 && str[0].type === "space" && str[0].value === "partition" && str[1].type === "space" && str[1].value === "by") { + str = str.slice(2); + if (str.length > 0 && str[0].type === "bracket") { + result[4] = buildSplitString(str[0].value); + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "limit") { + str = str.slice(1); + if (str.length > 0) { + if (str[0].type !== "bracket") { + result[2] = parseInt(str[0].value, 10); + } + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "offset") { + str = str.slice(1); + if (str.length > 0) { + if (str[0].type !== "bracket") { + result[3] = parseInt(str[0].value, 10); + } + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "desc") { + result[1] = true; + str.shift(); + } else { + break; + } + } + } + return { params: result, length: length - str.length }; + }, + }, { + id: VictoriaLogsOperationId.Split, + name: 'Split', + params: [{ + name: "Seperator", + type: "string", + }, { + name: "src field", + type: "string", + editor: FieldEditor, + }, + { + name: "dst field", + type: "string", + editor: FieldEditor, + }], + defaultParams: [",", "_msg", "_msg"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const seperator = model.params[0] as string; + const srcField = model.params[1] as string; + const dstField = model.params[2] as string; + let expr = `split ${quoteString(seperator, false)}` + if (srcField !== "_msg") { + expr += ` at ${quoteString(srcField)}` + } + if (dstField !== "_msg") { + expr += ` as ${quoteString(dstField)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, string, string] = [",", "_msg", "_msg"]; + if (str.length > 0) { + if (isValue(str[0])) { + params[0] = getValue(str[0]); + str = str.slice(1); + } else { + return { params, length: length - str.length }; + } + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "from") { + str.shift(); + if (isValue(str[0])) { + params[1] = getValue(str[0]); + str.shift(); + } + } + } + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "as") { + str.shift(); + if (isValue(str[0])) { + params[2] = getValue(str[0]); + str.shift(); + } + } + } + } + return { params, length: length - str.length }; + } + }, + { + id: VictoriaLogsOperationId.Stats, + name: 'Stats', + params: [{ + name: "Stats by", + type: "string", + }, { + name: "Stats", + type: "string", + editor: StatsEditor("stats"), + }], + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: renderStatsPipe, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsPipe, + }, + { + id: VictoriaLogsOperationId.StreamContext, + name: 'Stream context', + params: [{ + name: "Before", + type: "number", + }, { + name: "After", + type: "number", + }, { + name: "Time window", + type: "string", + }], + defaultParams: [0, 0, "1h"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const before = model.params[0] as number; + const after = model.params[1] as number; + const time_window = model.params[2]; + let expr = "stream_context"; + if (before > 0) { + expr += " before " + before; + } + if (after > 0) { + expr += " after " + after; + } + if (time_window !== "1h" && time_window !== "") { + expr += " time_window " + time_window; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let before = 0; + let after = 0; + let length = str.length; + let time_window = "1h"; + let i = 0; + while (str.length >= 2) { + if (i === 3) { + break; // max 3 params + } + if (str[0].type === "space" && str[0].value === "before") { + str = str.slice(1); + if (isValue(str[0])) { + before = parseInt(getValue(str[0]), 10); + } + i++; + str.shift(); + } else if (str[0].type === "space" && str[0].value === "after") { + str = str.slice(1); + if (isValue(str[0])) { + after = parseInt(getValue(str[0]), 10); + } + i++; + str.shift(); + } else if (str[0].type === "space" && str[0].value === "time_window") { + str = str.slice(1); + if (isValue(str[0])) { + time_window = getValue(str[0]); + } + i++; + str.shift(); + } else { + break; + } + } + return { params: [before, after, time_window], length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.TimeAdd, + name: 'Time add', + params: [{ + name: "Duration", + type: "string", + }, { + name: "Field", + type: "string", + editor: FieldEditor, + }], + alternativesKey: "time", + defaultParams: ["1h", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const duration = model.params[0] as string; + const field = model.params[1] as string; + let expr = `time_add ${duration}`; + if (field !== "") { + expr += ` at ${quoteString(field)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + const length = str.length; + const params: [string, string] = ["1h", ""]; + if (str.length > 0) { + if (str[0].type === "quote") { + params[0] = str[0].value; + str.shift(); + } else if (str[0].type === "space") { + params[0] = str[0].value; + str.shift(); + if (str.length > 0 && params[0] === "-") { + params[0] += str[0].value; + str.shift(); + } + } else { + return { params, length: length - str.length }; + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "at") { + str.shift(); + if (str.length > 0 && isValue(str[0])) { + params[1] = getValue(str[0]); + str.shift(); + } + } + } + return { params, length: length - str.length }; + } + }, + { + id: VictoriaLogsOperationId.Top, + name: 'Top', + params: [{ + name: "Top Number", + type: "number", + }, { + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Hits field name", + type: "string", + }, { + name: "Add rank", + type: "boolean", + }, { + name: "Rank field name", + type: "string", + editor: ResultFieldEditor, + }], + alternativesKey: "reduce", + defaultParams: [10, "", "", false, "rank"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const topNumber = model.params[0] as number; + const fields = model.params[1] as string; + const hitsFieldName = model.params[2] as string; + const addRank = model.params[3] as boolean; + const rankFieldName = model.params[4] as string; + let expr = "top "; + if (topNumber !== 10) { + expr += topNumber + " "; + } + expr += `by (${fields})`; + if (hitsFieldName !== "") { + expr += ` hits as ${quoteString(hitsFieldName)}`; + } + if (addRank) { + expr += " rank"; + if (rankFieldName !== "rank") { + expr += " as " + quoteString(rankFieldName); + } + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let result: [number, string, string, boolean, string] = [10, "", "", false, "rank"]; + if (str.length > 0) { + if (str[0].type === "space" || str[0].type === "quote") { + const value = getValue(str[0]).replace("_", ""); + if (!Number.isNaN(parseInt(value, 10))) { + result[0] = parseInt(value, 10); + str.shift(); + } + } + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "by") { + str = str.slice(1); + } + if (str[0].type === "bracket") { + result[1] = str[0].raw_value.slice(1, -1); + str.shift(); + } else if (isValue(str[0])) { + const values = getFieldList(str); + result[1] = values.join(", "); + } else { + return { params: result, length: length - str.length }; + } + } + let i = 0; + while (str.length > 0 && i < 2) { + if (str.length >= 3 && str[0].type === "space" && str[0].value === "hits" && str[1].type === "space" && str[1].value === "as") { + str = str.slice(2); + if (isValue(str[0])) { + result[2] = getValue(str[0]); + str.shift(); + i++; + } + } else if (str[0].type === "space" && (str[0].value === "rank" || str[0].value === "with")) { + if (str[0].value === "with") { + str.shift(); + } + result[3] = true; + str = str.slice(1); + i++; + if (str.length >= 2 && str[0].type === "space" && str[0].value === "as") { + str = str.slice(1); + if (isValue(str[0])) { + result[4] = getValue(str[0]); + str.shift(); + } + } + } else { + break; + } + } + } + return { params: result, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.TotalStats, + name: 'Total stats', + params: [{ + name: "Stats by", + type: "string", + }, { + name: "Stats", + type: "string", + editor: StatsEditor("total"), + }], + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: renderStatsPipe, + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsPipe, + }, + { + id: VictoriaLogsOperationId.Union, + name: 'Union', + params: [{ + name: "", + type: "string", + editor: QueryEditor, + }], + alternativesKey: "combine", + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const expr = "union (" + model.params[0] + ")"; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let query = ""; + const length = str.length; + if (str.length > 0) { + if (str[0].type === "bracket") { + query = str[0].raw_value.slice(1, -1); + str.shift(); + } + } + return { params: [query], length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Uniq, + name: 'Uniq', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "With hits", + type: "boolean", + }, { + name: "Limit", + type: "number", + }], + alternativesKey: "reduce", + defaultParams: ["", false, 0], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const withHits = model.params[1] as boolean; + const limit = model.params[2] as number; + let expr = `uniq (${fields})`; + if (limit > 0) { + expr += " limit " + limit; + } + if (withHits) { + expr += " with hits"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let withHits = false; + let fields = ""; + let limit = 0; + let Length = str.length; + + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "by") { + str.shift(); + } + } + // Check for "by (fields)" + if (str.length > 0) { + if (str[0].type === "bracket") { + fields = str[0].raw_value.slice(1, -1); + str = str.slice(1); + } else if (isValue(str[0])) { + let values = getFieldList(str); + fields = values.join(", "); + } else { + return { + params: [fields, withHits, limit], + length: Length - str.length, + } + } + } + let i = 0; + while (str.length >= 2) { + if (i === 2) { + break; // max 2 params + } else if ( + // Check for "with hits" + str[0].type === "space" && str[0].value === "with" && + str[1].type === "space" && str[1].value === "hits" + ) { + withHits = true; + str = str.slice(2); + } else if (str[0].type === "space" && str[0].value === "limit") { + // uniq by (host, path) limit 100 + str = str.slice(1); + if (isValue(str[0])) { + limit = parseInt(getValue(str[0]), 10); + } + str.shift(); + } + i++; + } + return { + params: [fields, withHits, limit], + length: Length - str.length, + } + }, + }, + { + id: VictoriaLogsOperationId.UnpackJson, + name: 'Unpack JSON', + params: [{ + name: "Unpack from field", + type: "string", + editor: FieldEditor, + }, { + name: "Fields to unpack", + type: "string", + editor: UnpackedFieldsSelector("unpack_json"), + }, { + name: "Result prefix", + type: "string", + }, { + name: "Keep original fields", + type: "boolean", + }, { + name: "Skip empty results", + type: "boolean", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "unpack", + defaultParams: [this.defaultField, "", "", "", false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fromField = model.params[0] as string; + const fields = model.params[1] as string; + const resultPrefix = model.params[2] as string; + const keepOriginalFields = model.params[3] as boolean; + const skipEmptyResults = model.params[4] as boolean; + const condition = model.params[5] as string; + let expr = "unpack_json"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (fromField !== this.defaultField) { + expr += ` from ${quoteString(fromField)}`; + } + if (fields !== "") { + expr += ` fields (${fields})`; + } + if (resultPrefix !== "") { + expr += ` result_prefix ${quoteString(resultPrefix)}`; + } + if (keepOriginalFields) { + expr += " keep_original_fields"; + } + if (skipEmptyResults) { + expr += " skip_empty_results"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let len = str.length; + const condition = getConditionFromString(str); + const conLength = len - str.length; + const { fromField, fields, resultPrefix, keepOriginalFields, skipEmptyResults, length } = this.parseUnpackPipe(str); + return { + params: [fromField, fields, resultPrefix, keepOriginalFields, skipEmptyResults, condition], + length: conLength + length, + } + }, + }, + { + id: VictoriaLogsOperationId.UnpackLogfmt, + name: 'Unpack logfmt', + params: [{ + name: "Unpack from field", + type: "string", + editor: FieldEditor, + }, { + name: "Fields to unpack", + type: "string", + editor: UnpackedFieldsSelector("unpack_logfmt"), + }, { + name: "Result prefix", + type: "string", + }, { + name: "Keep original fields", + type: "boolean", + }, { + name: "Skip empty results", + type: "boolean", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "unpack", + defaultParams: [this.defaultField, "", "", "", false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fromField = model.params[0] as string; + const fields = model.params[1] as string; + const resultPrefix = model.params[2] as string; + const keepOriginalFields = model.params[3] as boolean; + const skipEmptyResults = model.params[4] as boolean; + const condition = model.params[5] as string; + let expr = "unpack_logfmt"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (fromField !== this.defaultField) { + expr += ` from ${quoteString(fromField)}`; + } + if (fields !== "") { + expr += ` fields (${fields})`; + } + if (resultPrefix !== "") { + expr += ` result_prefix ${quoteString(resultPrefix)}`; + } + if (keepOriginalFields) { + expr += " keep_original_fields"; + } + if (skipEmptyResults) { + expr += " skip_empty_results"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let len = str.length; + const condition = getConditionFromString(str); + const conLength = len - str.length; + const { fromField, fields, resultPrefix, keepOriginalFields, skipEmptyResults, length } = this.parseUnpackPipe(str); + return { + params: [fromField, fields, resultPrefix, keepOriginalFields, skipEmptyResults, condition], + length: conLength + length, + } + }, + }, + { + id: VictoriaLogsOperationId.UnpackSyslog, + name: 'Unpack syslog', + params: [{ + name: "Unpack from field", + type: "string", + editor: FieldEditor, + }, { + name: "Fields to unpack", + type: "string", + editor: UnpackedFieldsSelector("unpack_syslog"), + }, { + name: "Result prefix", + type: "string", + }, { + name: "Keep original fields", + type: "boolean", + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }, { + name: "Time offset", + type: "string", + }], + alternativesKey: "unpack", + defaultParams: [this.defaultField, "", "", false, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fromField = model.params[0] as string; + const fields = model.params[1] as string; + const resultPrefix = model.params[2] as string; + const keepOriginalFields = model.params[3] as boolean; + const condition = model.params[4] as string; + const timeOffset = model.params[5] as string; + let expr = "unpack_syslog"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (fromField !== this.defaultField) { + expr += ` from ${quoteString(fromField)}`; + } + if (fields !== "") { + expr += ` fields (${fields})`; + } + if (resultPrefix !== "") { + expr += ` result_prefix ${quoteString(resultPrefix)}`; + } + if (keepOriginalFields) { + expr += " keep_original_fields"; + } + if (timeOffset !== "") { + expr += " offset " + timeOffset; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let len = str.length; + const condition = getConditionFromString(str); + const conLength = len - str.length; + const { fromField, fields, resultPrefix, keepOriginalFields, offset, length } = this.parseUnpackPipe(str); + return { + params: [fromField, fields, resultPrefix, keepOriginalFields, condition, offset], + length: conLength + length, + }; + }, + }, + { + id: VictoriaLogsOperationId.UnpackWords, + name: 'Unpack words', + params: [{ + name: "From field", + type: "string", + editor: FieldEditor, + }, { + name: "Dst field", + type: "string", + }, { + name: "Drop duplicates", + type: "boolean", + }], + alternativesKey: "unpack", + defaultParams: [this.defaultField, "", false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fromField = model.params[0] as string; + const dstField = model.params[1] as string; + const dropDuplicates = model.params[2] as boolean; + let expr = "unpack_words"; + expr += ` from ${quoteString(fromField)}`; + if (dstField !== "") { + expr += ` as ${quoteString(dstField)}`; + } + if (dropDuplicates) { + expr += " drop_duplicates"; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, string, boolean] = [this.defaultField, "", false]; + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "from") { + str.shift(); + } + if (str.length > 0 && isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "as") { + str = str.slice(1); + if (str.length > 0 && isValue(str[0])) { + params[1] = getValue(str[0]); + str.shift(); + } + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "drop_duplicates") { + params[2] = true; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Unroll, + name: 'Unroll', + params: [{ + name: "Unroll from fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Condition", + type: "string", + editor: this.conditionalEditor, + }], + alternativesKey: "unpack", + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Pipes, + renderer: (model, def, innerExpr) => { + const fields = model.params[0] as string; + const condition = model.params[1] as string; + let expr = "unroll"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (fields !== "") { + expr += ` (${fields})`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let params: [fields: string, condition: string] = ["", ""]; + let length = str.length; + params[1] = getConditionFromString(str); + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "by") { + str.shift(); + } + } + if (str.length > 0) { + if (str[0].type === "bracket") { + params[0] = str[0].raw_value.slice(1, -1); + str.shift(); + } else if (isValue(str[0])) { + let values = getFieldList(str); + params[0] = values.join(", "); + } + } + return { params, length: length - str.length }; + }, + } + ]; + } + + getStatsDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return [ + { + id: VictoriaLogsOperationId.Avg, + name: 'Avg', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Count, + name: 'Count', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.CountEmpty, + name: 'Count empty', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Limit", + type: "string", + editor: NumberEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", 0, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(true), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(true), + }, + { + id: VictoriaLogsOperationId.CountUniq, + name: 'Count uniq', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Limit", + type: "string", + editor: NumberEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", 0, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(true), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(true), + }, + { + id: VictoriaLogsOperationId.CountUniqHash, + name: 'Count uniq hash', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Histogram, + name: 'Histogram', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.JsonValues, + name: 'Json values', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Sort by", + type: "string", + editor: SortedFieldsEditor, + }, { + name: "Limit", + type: "string", + editor: NumberEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", 0, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(true, true), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(true, true), + }, + { + id: VictoriaLogsOperationId.Max, + name: 'Max', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Median, + name: 'Median', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Min, + name: 'Min', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Quantile, + name: 'Quantile', + params: [{ + name: "Percentile", + type: "number", + placeholder: "Nth", + description: "Percentile value (0-100)", + }, { + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: [5, "", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: (model, def, innerExpr) => { + const percentile = model.params[0] as number; + const fields = model.params[1] as string; + const resultField = model.params[2] as string; + const condition = model.params[3] as string; + let expr = `quantile(0.${percentile}, ${fields})`; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (resultField !== "") { + expr += ` ${quoteString(resultField)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let params: [number, string, string, string] = [5, "", "", ""]; + const length = str.length; + if (str.length > 0) { + if (str[0].type === "space") { + str.shift(); + } + if (str[0].type === "bracket") { + const value = str[0].value; + if (value.length > 0) { + const phi = value.shift(); + if (phi) { + let percentile = getValue(phi); + percentile = percentile.replace(/^0\./, ""); + params[0] = Number.parseInt(percentile, 10); + } + if (value.length > 0 && value[0].value === ",") { + value.shift(); + } + params[1] = buildSplitString(value); + } + str.shift(); + } + params[3] = getConditionFromString(str); + params[2] = getFieldValue(str); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Rate, + name: 'Rate', + params: [{ + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: (model, def, innerExpr) => { + const resultField = model.params[0] as string; + const condition = model.params[1] as string; + let expr = "rate()"; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (resultField !== "") { + expr += ` ${quoteString(resultField)}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let params: [string, string] = ["", ""]; + const length = str.length; + if (str.length > 0) { + if (str[0].type === "bracket" && str[0].prefix === "rate") { + str.shift(); + } else if (str[0].type === "space" && str[0].value === "rate") { + str = str.slice(2); + } + } + params[1] = getConditionFromString(str); + params[0] = getFieldValue(str); + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.RateSum, + name: 'Rate sum', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.RowAny, + name: 'Row any', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.RowMax, + name: 'Row max', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.RowMin, + name: 'Row min', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.Sum, + name: 'Sum', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.SumLen, + name: 'Sum len', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + }, + { + id: VictoriaLogsOperationId.UniqValues, + name: 'Uniq values', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Limit", + type: "string", + editor: NumberEditor, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", 0, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(true), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(true), + }, + { + id: VictoriaLogsOperationId.Values, + name: 'Values', + params: [{ + name: "Fields", + type: "string", + editor: FieldsEditorWithPrefix, + }, { + name: "Result field", + type: "string", + editor: ResultFieldEditor, + }, { + name: 'Condition', + type: 'string', + editor: this.conditionalEditor, + }], + alternativesKey: "stats", + defaultParams: ["", "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Stats, + renderer: renderStatsOperation(false), + addOperationHandler: addVictoriaOperation, + splitStringByParams: parseStatsOperation(false), + } + ]; + } + + getSpecialDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return [{ + id: VictoriaLogsOperationId.Options, + name: 'Options', + params: [{ + name: "Query concurrency", + type: "number", + }, { + name: "Parallel readers", + type: "number", + }, { + name: "Ignore global time filter", + type: "boolean", + }, { + name: "Time offset", + type: "string", + }, { + name: "Allow partial response", + type: "boolean", + }], + defaultParams: [0, 0, false, "", false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Special, + renderer: (model, def, innerExpr) => { + const queryConcurrency = model.params[0] as number; + const parallelReaders = model.params[1] as number; + const ignoreGlobalTimeFilter = model.params[2] as boolean; + const timeOffset = model.params[3] as string; + const allowPartialResponse = model.params[4] as boolean; + let expr = "options("; + if (queryConcurrency > 0) { + expr += "concurrency=" + queryConcurrency; + } + if (parallelReaders > 0) { + if (expr.length > 8) { + expr += ", "; + } + expr += "parallel_readers=" + parallelReaders; + } + if (ignoreGlobalTimeFilter) { + if (expr.length > 8) { + expr += ", "; + } + expr += "ignore_global_time_filter=true"; + } + if (timeOffset !== "") { + if (expr.length > 8) { + expr += ", "; + } + expr += "time_offset=" + timeOffset; + } + if (allowPartialResponse) { + if (expr.length > 8) { + expr += ", "; + } + expr += "allow_partial_response=true"; + } + expr += ")"; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + const length = str.length; + const params: [number, number, boolean, string, boolean] = [0, 0, false, "", false]; + if (str.length > 0) { + if (str[0].value === "options") { + str.shift(); + } + if (str.length > 0 && str[0].type === "bracket") { + for (const value of str[0].raw_value.slice(1, -1).split(",")) { + const trimmedValue = value.trim(); + if (trimmedValue.startsWith("concurrency")) { + const parts = trimmedValue.split("="); + if (parts.length === 2) { + params[0] = parseInt(parts[1], 10); + } + } else if (trimmedValue.startsWith("parallel_readers")) { + const parts = trimmedValue.split("="); + if (parts.length === 2) { + params[1] = parseInt(parts[1], 10); + } + } else if (trimmedValue.startsWith("ignore_global_time_filter")) { + if (trimmedValue.endsWith("true")) { + params[2] = true; + } else if (trimmedValue.endsWith("false")) { + params[2] = false; + } + } else if (trimmedValue.startsWith("time_offset")) { + const parts = trimmedValue.split("="); + if (parts.length === 2) { + params[3] = parts[1]; + } + } else if (trimmedValue.startsWith("allow_partial_response")) { + if (trimmedValue.endsWith("true")) { + params[4] = true; + } else if (trimmedValue.endsWith("false")) { + params[4] = false; + } + } + } + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, { + id: VictoriaLogsOperationId.FieldContainsAnyValueFromVariable, + name: 'Field contains any value from Variable', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Variable", + type: "string", + editor: VariableEditor, + }], + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Special, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const variable = model.params[1] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = quoteString(field) + ":"; + } + expr += `(${variable})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string] = ["", ""]; + params[0] = fieldName || this.defaultField; + if (str.length > 0) { + if (str[0].type === "bracket") { + params[1] = str[0].raw_value.slice(1, -1); + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, { + id: VictoriaLogsOperationId.Comment, + name: 'Comment', + params: [{ + name: "Comment", + type: "string", + }], + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Special, + renderer: (model, def, innerExpr) => { + const comment = model.params[0] as string; + const expr = `# ${comment} \n`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let params: string[] = [""]; + if (str.length > 0 && str[0].type === "comment") { + params[0] = str[0].value; + } + return { params, length: 1 }; + } + }]; + } + + getOperationDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return [ + ...this.getFilterDefinitions(), + ...this.getPipeDefinitions(), + ...this.getStatsDefinitions(), + ...this.getSpecialDefinitions(), + ]; + } + + parsePackPipe(str: SplitString[]) { + let length = str.length; + let params: [string, string] = ["", this.defaultField]; + if (str.length === 0) { + return { params, length: 0 }; + } + // fields (foo, bar) as baz + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "fields") { + str = str.slice(1); + if (str[0].type === "bracket") { + params[0] = str[0].raw_value.slice(1, -1); + str.shift(); + } else { + return { params, length: length - str.length }; + } + } + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "as") { + str = str.slice(1); + if (str.length > 0 && isValue(str[0])) { + params[1] = getValue(str[0]); + str.shift(); + } + } + return { params, length: length - str.length }; + } + + parseUnpackPipe(str: SplitString[]) { + const strLen = str.length; + /// (without the pipe commands) + + // unpack_json + // unpack_json from _msg + // unpack_json from my_json fields (foo, bar) + // unpack_json from foo fields (ip, host) keep_original_fields + // unpack_json fields (ip, host) skip_empty_results + // unpack_json from foo result_prefix "foo_" + + // unpack_logfmt from foo result_prefix "foo_" + // unpack_logfmt fields (ip, host) skip_empty_results + // unpack_logfmt from foo fields (ip, host) keep_original_fields + // unpack_logfmt from my_logfmt fields (foo, bar) + // unpack_logfmt from _msg + + // unpack_syslog keep_original_fields + // unpack_syslog from foo result_prefix "foo_" + // unpack_syslog offset 5h30m + + let fromField = this.defaultField; + let fields = ""; + let resultPrefix = ""; + let keepOriginalFields = false; + let skipEmptyResults = false; + let offset = ""; + while (str.length > 0) { + if (str[0].type === "space" && str[0].value === "from") { + str = str.slice(1); + if (isValue(str[0])) { + fromField = getValue(str[0]); + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "fields") { + str = str.slice(1); + if (str[0].type === "bracket") { + fields = str[0].raw_value.slice(1, -1); + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "keep_original_fields") { + keepOriginalFields = true; + str.shift(); + } else if (str[0].type === "space" && str[0].value === "skip_empty_results") { + skipEmptyResults = true; + str.shift(); + } else if (str[0].type === "space" && str[0].value === "offset") { + str = str.slice(1); + if (isValue(str[0])) { + offset = getValue(str[0]); + str.shift(); + } + } else if (str[0].type === "space" && str[0].value === "result_prefix") { + str = str.slice(1); + if (str[0].type === "quote") { + resultPrefix = unquoteString(str[0].value); + str.shift(); + } + } else { + break; + } + } + return { fromField, fields, resultPrefix, keepOriginalFields, skipEmptyResults, offset, length: strLen - str.length }; + } + + parseCommonCase(str: SplitString[], fieldName?: string): { params: QueryBuilderOperationParamValue[], length: number } { + const length = str.length; + const params: string[] = [fieldName || this.defaultField]; + if (str.length > 0) { + if (str[0].type === "space" && (str[0].value === "equals_common_case" || str[0].value === "contains_common_case")) { + str = str.slice(1); + if (str[0].type === "bracket") { + params.push(...getValuesFromBrackets(str[0].value)); + str.shift(); + } + } else if (str[0].type === "bracket" && (str[0].prefix === "equals_common_case" || str[0].prefix === "contains_common_case")) { + if (str[0].type === "bracket") { + params.push(...getValuesFromBrackets(str[0].value)); + str.shift(); + } + } + } + if (params.length === 1) { + params.push(""); + } + return { + params, length: length - str.length + } + } + + renderCommonCase(model: QueryBuilderOperation, def: QueryBuilderOperationDefinition, innerExpr: string) { + const field = model.params[0] as string; + const params = model.params.slice(1).filter((v) => Boolean(v) || v === "") as string[]; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + if (params.length > 0) { + let values = ""; + for (let i = 0; i < params.length; i++) { + let value = params[i]; + if (value.startsWith("$") || value === "*") { + value = value; + } else { + value = quoteString(value, false); + } + values += value; + if (i < params.length - 1) { + values += ", "; + } + } + expr += model.id + `(${values})`; + } else { + expr += model.id + `("")`; + } + return pipeExpr(innerExpr, expr); + } + + getFilterDefinitions(): VictoriaQueryBuilderOperationDefinition[] { + return [ + { + id: VictoriaLogsOperationId.Word, + name: 'Word', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Word", + type: "string", + }, { + name: "Case Insensitive", + type: "boolean", + }, { + name: "prefix", + type: "boolean", + }], + defaultParams: [this.defaultField, "", false, false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + explainHandler: () => `[word-filter](https://docs.victoriametrics.com/victorialogs/logsql/#word-filter) \\\n use \`*\` for non-empty field`, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const word = model.params[1] as string; + const caseInsensitive = model.params[2] as boolean; + const prefix = model.params[3] as boolean; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + let wordValue = quoteString(word); + if (word.startsWith("$") || word === "*") { + wordValue = word; + } + if (wordValue === "") { + wordValue = '""'; + } + if (prefix) { + wordValue += "*"; + } + if (caseInsensitive) { + expr += `i(${wordValue})`; + } else { + expr += `${wordValue}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params = ["", "", false]; + params[0] = fieldName || this.defaultField; + + if (str.length === 0) { // value + + } else if (str[0].type === "colon" && str[0].value !== "") { // shouldn't be + params[1] = str[0].value; + } else if (str[0].type === "quote") { // "value" + params[1] = unquoteString(str[0].value); + } else if (str[0].type === "space") { // value + params[1] = str[0].value; + } else if (str[0].type === "bracket" && str[0].prefix === "i") { // i("value") / i(value) + const value = str[0].value; + if (value[0].type === "space") { + params[1] = value[0].value; + } else if (value[0].type === "quote") { + params[1] = unquoteString(value[0].value); + } + params[2] = true; // case insensitive + } else if (str[0].type === "bracket" && str[0].prefix === "") { // ("value") / (value) + const value = str[0].value; + if (isValue(value[0])) { + params[1] = getValue(value[0]); + } + if (value.length > 1 && value[1].value === "*") { + params[3] = true; + } + } + str.shift(); + if (str.length > 0 && str[0].type === "space" && str[0].value === "*") { + params[3] = true; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Time, + name: 'Time filter', + params: [{ + name: "Time Filter", + type: "string", + }, { + name: "Time offset", + type: "string", + }], + defaultParams: ["", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const timeFilter = model.params[0] as string; + const offset = model.params[1] as string; + let expr = "_time:" + timeFilter; + if (offset !== "") { + expr += ` offset ${offset}`; + } + return pipeExpr(innerExpr, expr); + }, + explainHandler: () => `[time-filter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter)`, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let filter = ""; + let offset = ""; + if (str.length === 0) { + + } else if (str[0].type === "bracket") { // [min_time, max_time) + filter = str[0].raw_value; + str.shift(); + } else if (str[0].type === "space") { // min_time + filter = str[0].value; + str.shift(); + } else if (str[0].type === "colon" && str[0].value !== "_time") { // YYYY-MM-DDTHH:MM:SSZ + let values: string[] = [] + do { + values.push(str[0].value as string); + } while (str[0].type === "colon" && (str = str.slice(1))) + str.shift(); + filter = values.join(":") + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "offset") { + str = str.slice(1); + if (isValue(str[0])) { + offset = getValue(str[0]); + str.shift(); + } + } + return { params: [filter, offset], length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.DayRange, + name: 'Day range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Start", + type: "string", + placeholder: "HH:MM", + }, { + name: "End", + type: "string", + placeholder: "HH:MM", + }, { + name: "Include start", + type: "boolean", + }, { + name: "Include end", + type: "boolean", + }, { + name: "Offset", + type: "string", + }], + defaultParams: ["_time", "08:00", "18:00", false, false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const start = model.params[1] as string; + const end = model.params[2] as string; + const includeStart = model.params[3] as boolean; + const includeEnd = model.params[4] as boolean; + const offset = model.params[5] as string; + let expr = `${field}:day_range`; + expr += includeStart ? "[" : "(" + expr += `${start}, ${end}`; + expr += includeEnd ? "]" : ")"; + if (offset !== "") { + expr += ` offset ${offset}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let params: [string, string, string, boolean, boolean, string] = ["_time", "08:00", "18:00", false, false, ""]; + let length = str.length; + params[0] = fieldName || "_time"; + if (str.length > 0) { + if (str[0].type === "space") { + str.shift(); + } + if (str.length > 0 && str[0].type === "bracket") { + const raw_value = str[0].raw_value; + if (raw_value.startsWith("[")) { + params[3] = true; + } + if (raw_value.endsWith("]")) { + params[4] = true; + } + const value = raw_value.slice(1, -1).split(","); + if (value.length === 2) { + params[1] = value[0].trim(); + params[2] = value[1].trim(); + } + str.shift(); + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "offset") { + str = str.slice(1); + if (isValue(str[0])) { + params[5] = getValue(str[0]); + str.shift(); + } + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.WeekRange, + name: 'Week range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Start", + type: "string", + options: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"], + }, { + name: "End", + type: "string", + options: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"], + }, { + name: "Include start", + type: "boolean", + }, { + name: "Include end", + type: "boolean", + }, { + name: "Offset", + type: "string", + }], + defaultParams: ["_time", "Mon", "Fri", false, false, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const start = model.params[1] as string; + const end = model.params[2] as string; + const includeStart = model.params[3] as boolean; + const includeEnd = model.params[4] as boolean; + const offset = model.params[5] as string; + let expr = `${field}:week_range`; + expr += includeStart ? "[" : "(" + expr += `${start}, ${end}`; + expr += includeEnd ? "]" : ")"; + if (offset !== "") { + expr += ` offset ${offset}`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let params: [string, string, string, boolean, boolean, string] = ["_time", "Mon", "Fri", false, false, ""]; + let length = str.length; + params[0] = fieldName || "_time"; + if (str.length > 0) { + if (str[0].type === "space") { + str.shift(); + } + if (str.length > 0 && str[0].type === "bracket") { + const raw_value = str[0].raw_value; + if (raw_value.startsWith("[")) { + params[3] = true; + } + if (raw_value.endsWith("]")) { + params[4] = true; + } + const value = raw_value.slice(1, -1).split(","); + if (value.length === 2) { + let startDay = value[0].trim(); + let endDay = value[1].trim(); + const daysOfWeek = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]; + if (startDay.length === 3) { + startDay = daysOfWeek.find(day => day.startsWith(startDay)) || startDay; + } + if (endDay.length === 3) { + endDay = daysOfWeek.find(day => day.startsWith(endDay)) || endDay; + } + params[1] = startDay; + params[2] = endDay; + } + str.shift(); + } + if (str.length > 0 && str[0].type === "space" && str[0].value === "offset") { + str = str.slice(1); + if (isValue(str[0])) { + params[5] = getValue(str[0]); + str.shift(); + } + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Stream, + name: 'Stream', + params: [{ + name: "Field", + type: "string", + restParam: true, + editor: StreamFieldEditor, + }], + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const labels = model.params.join(", "); + const expr = `{${labels}}` + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: string[] = []; + if (str.length > 0) { + if (str[0].type === "bracket") { + for (const parts of splitByUnescapedChar(str[0].value, ",")) { + params.push(buildSplitString(parts)); + } + const value = str[0].value; + if (value.length > 0 && value[value.length - 1].value === ",") { + params.push(""); + } + str.shift(); + } + } + if (params.length === 0) { + params[0] = ""; + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.StreamId, + name: 'Stream ID', + params: [{ + name: "", + type: "string", + editor: SubqueryEditor, + }], + defaultParams: [""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const subqueryField = model.params[0] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + const expr = `_stream_id:${subquery}`; + return pipeExpr(innerExpr, expr); + }, + explainHandler: () => `[stream-id-filter](https://docs.victoriametrics.com/victorialogs/logsql/#_stream_id-filter)`, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string] = [""]; + if (str.length > 0) { + if (isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } else if (str[0].type === "bracket") { + params[0] = str[0].raw_value; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Regexp, + name: 'Regexp', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Expression", + type: "string", + placeholder: "" + }, { + name: "Case Insensitive", + type: "boolean", + }], + defaultParams: [this.defaultField, "", false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const substr = model.params[1] as string; + const caseInsensitive = model.params[2] as boolean; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + if (caseInsensitive) { + expr += `~${quoteString("(?i)" + substr, true)}`; + } else { + expr += `~${quoteString(substr, true)}`; + } + return pipeExpr(innerExpr, expr); + }, + explainHandler: () => `[regexp-filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter)`, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [fieldName: string, substr: string, caseInsensitive: boolean] = ["", "", false]; + params[0] = fieldName || this.defaultField; + if (str.length >= 2) { + if (str[0].value === "~") { + str = str.slice(1); + if (str[0].type === "quote") { + let substr = unquoteString(str[0].value); + if (substr.startsWith("(?i)")) { + params[2] = true; + substr = substr.slice("(?i)".length); + } + params[1] = substr; + str.shift(); + } + } + } else if (str.length > 0) { + if (str[0].value === "~") { + str.shift(); + } else { + + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.PatternMatch, + name: 'Pattern match', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Pattern", + type: "string", + placeholder: "" + }, { + name: "Full match", + type: "boolean", + }], + defaultParams: [this.defaultField, "", false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const pattern = model.params[1] as string; + const fullMatch = model.params[2] as boolean; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + if (fullMatch) { + expr += "pattern_match_full" + } else { + expr += "pattern_match" + } + expr += `(${quoteString(pattern, true)})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + const length = str.length; + const params: [string, string, boolean] = [fieldName || this.defaultField, "", false]; + if (str.length > 0) { + if (str[0].type === "space") { + params[2] = str[0].value === "pattern_match_full"; + str = str.slice(1); + } else if (str[0].type === "bracket") { + params[2] = str[0].prefix === "pattern_match_full"; + } else { + return { params, length: length - str.length }; + } + if (str.length > 0 && str[0].type === "bracket") { + params[1] = (str[0].value.length && isValue(str[0].value[0])) ? getValue(str[0].value[0]) : ""; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Substring, + name: 'Substring', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Substring", + type: "string", + }], + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const substr = model.params[1] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + if (substr.startsWith("$")) { + expr += `*${substr}*`; + } else if (substr === "") { + expr += '*""*'; + } else { + expr += `*${quoteString(substr)}*`; + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + let length = str.length; + let params: [string, string] = [this.defaultField, ""]; + if (str.length > 2 && str[0].value === "*" && str[1].type === "quote" && str[2].value === "*") { + params[1] = getValue(str[1]); + str = str.slice(3); + } else if (str.length > 0) { + if (str[0].type === "space") { + const value = str[0].value; + if (value.startsWith("*") && value.endsWith("*")) { + params[1] = value.slice(1, -1); + str.shift(); + } + } + } + return { params, length: length - str.length }; + } + }, + { + id: VictoriaLogsOperationId.RangeComparison, + name: 'Range comparison', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Comparison", + type: "string", + options: [ + { label: "equal", value: "=" }, + { label: "less than", value: "<" }, + { label: "less than or equal", value: "<=" }, + { label: "greater than", value: ">" }, + { label: "greater than or equal", value: ">=" }, + ], + }, { + name: "Value", + type: "string", + editor: NumberEditor, + }], + defaultParams: [this.defaultField, ">", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const comparison = model.params[1] as string; + const value = model.params[2] as string; + const expr = `${quoteString(field)}:${comparison}${value}`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let params = ["", "=", ""]; + params[0] = fieldName || this.defaultField; + let length = str.length; + if (str.length > 0 && str[0].type === "space") { + if (["<=", ">="].includes(str[0].value.slice(0, 2))) { + params[1] = str[0].value.slice(0, 2); + if (str[0].value.length > 2) { + params[2] = str[0].value.slice(2); + str.shift(); + } else if (str.length > 1 && str[1].type === "space") { + params[2] = str[1].value; + str.shift(); + str.shift(); + } + } else if (["<", ">", "="].includes(str[0].value.slice(0, 1))) { + params[1] = str[0].value.slice(0, 1); + if (str[0].value.length > 1) { + params[2] = str[0].value.slice(1); + str.shift(); + } else if (str.length > 1 && isValue(str[1])) { + params[2] = getValue(str[1]); + str.shift(); + str.shift(); + } else { + str.shift(); + } + } + } + return { params: params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Exact, + name: 'Exact', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Text", + type: "string", + editor: ExactValueEditor, + }, { + name: "Not equal", + type: "boolean", + }, { + name: "Exact Prefix", + type: "boolean", + }], + defaultParams: [this.defaultField, "", false, false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const text = model.params[1] as string; + const notEqual = model.params[2] as boolean; + const exactPrefix = model.params[3] as boolean; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += notEqual ? "!=" : "="; + if (text.startsWith("$")) { + expr += text; // variable + } else if (text === "") { + expr += '""'; // empty string + } else { + expr += quoteString(text); + } + if (exactPrefix) { + expr += "*"; // exact prefix + } + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string, boolean, boolean] = ["", "", false, false]; + params[0] = fieldName || this.defaultField; + if (str.length > 0 && str[0].type === "space") { // : = "server01" + if (str[0].value === "!") { + params[2] = true; // not equal + str.shift(); + } + if (str[0].value === "=") { + str = str.slice(1); + } + if (!isValue(str[0])) { + return { params, length: length - str.length }; + } + if (str[0].type === "quote") { + params[1] = unquoteString(str[0].value); + str = str.slice(1); + if (str.length > 0 && str[0].type === "space" && str[0].value === "*") { + params[3] = true; // exact prefix + str.shift(); + } + } else if (str[0].type === "space") { + let value = str[0].value; + if (value.endsWith("*")) { + params[3] = true; // exact prefix + value = value.slice(0, -1); + } + params[1] = value; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.MultiExact, + name: 'Multi exact', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Matches", + type: "string", + editor: SubqueryEditor, + }], + defaultParams: [this.defaultField, "()"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + let result = ""; + const fieldName = model.params[0] as string; + const subqueryField = model.params[1] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + if (fieldName !== this.defaultField) { + result = `${quoteString(fieldName)}:`; + } + result += `in${subquery}`; + return pipeExpr(innerExpr, result); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: string[] = ["", ""]; + params[0] = fieldName || this.defaultField; + if (str[0].type === "bracket") { // (="error" OR ="fatal") / in("error", "fatal") + params[1] = str[0].raw_value; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.ContainsAll, + name: 'Contains all', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Values", + type: "string", + editor: SubqueryEditor, + }], + defaultParams: [this.defaultField, "()"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const subqueryField = model.params[1] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += "contains_all" + subquery; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: string[] = ["", ""]; + params[0] = fieldName || this.defaultField; + if (str[0].type === "bracket" && str[0].prefix === "contains_all") { // contains_all(foo, "bar baz") + params[1] = str[0].raw_value; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.ContainsAny, + name: 'Contains any', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Filter", + type: "string", + editor: SubqueryEditor, + }], + defaultParams: [this.defaultField, "()"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const subqueryField = model.params[1] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += "contains_any" + subquery; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string] = [fieldName || this.defaultField, ""]; + if (str.length > 0 && str[0].type === "bracket" && str[0].prefix === "contains_any") { + params[1] = str[0].raw_value; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.EqualsCommonCase, + name: 'Equals common case', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Value", + type: "string", + restParam: true, + }], + alternativesKey: "commonCase", + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (...args) => this.renderCommonCase(...args), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (...args) => this.parseCommonCase(...args), + }, + { + id: VictoriaLogsOperationId.ContainsCommonCase, + name: 'Contains common case', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Value", + type: "string", + restParam: true, + }], + alternativesKey: "commonCase", + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (...args) => this.renderCommonCase(...args), + addOperationHandler: addVictoriaOperation, + splitStringByParams: (...args) => this.parseCommonCase(...args), + }, + { + id: VictoriaLogsOperationId.Sequence, + name: 'Sequence', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Sequence", + type: "string", + restParam: true, + }], + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const params = model.params.slice(1).filter((v) => Boolean(v) || v === "") as string[]; + let sequence = ""; + for (let i = 0; i < params.length; i++) { + sequence += quoteString(params[i]); + if (params[i] === "") { + sequence += '""'; + } + if (i < params.length - 1) { + sequence += ", "; + } + } + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `seq(${sequence})`; // seq("foo", "bar baz") + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: string[] = [""]; + params[0] = fieldName || this.defaultField; + if (str[0].type === "bracket" && str[0].prefix === "seq") { // seq(foo, "bar baz") + params.push(...getValuesFromBrackets(str[0].value)); + str.shift(); + } + if (params.length === 1) { + params[1] = ""; + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.Range, + name: 'Range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Lower", + type: "string", + editor: NumberEditor, + }, { + name: "Upper", + type: "string", + editor: NumberEditor, + }, { + name: "Include Lower", + type: "boolean", + }, { + name: "Include Upper", + type: "boolean", + }], + defaultParams: [this.defaultField, 0, 0, false, false], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const lower = model.params[1] as string; + const upper = model.params[2] as string; + const includeLower = model.params[3] as boolean; + const includeUpper = model.params[4] as boolean; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += "range"; + expr += includeLower ? "[" : "(" + expr += `${lower}, ${upper}`; + expr += includeUpper ? "]" : ")"; + return pipeExpr(innerExpr, expr); + }, + explainHandler: () => `[range-filter](https://docs.victoriametrics.com/victorialogs/logsql/#range-filterr)`, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string, string, boolean, boolean] = ["", "", "", false, false]; + params[0] = fieldName || this.defaultField; + if (str.length > 0) { // range(4.2, Inf) or >4.2 (... + if (str[0].type === "bracket" && str[0].prefix === "range") { // range(4.2, Inf) + const results = getValuesFromBrackets(str[0].value); + if (results.length === 2) { + params[1] = results[0]; + params[2] = results[1]; + if (str[0].raw_value.startsWith("[")) { + params[3] = true; + } + if (str[0].raw_value.endsWith("]")) { + params[4] = true; + } + } + str.shift(); + } else if (str[0].type === "space" && [">", ">=", "<", "<="].includes(str[0].value)) { // > 4.2 or >= 4.2 + params[0] = str[0].value; + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.IPv4Range, + name: 'IPv4 range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Start/CIDR", + type: "string", + }, { + name: "End", + type: "string", + }], + defaultParams: [this.defaultField, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const start = model.params[1] as string; + const end = model.params[2] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `ipv4_range(${start}${end ? `, ${end}` : ''})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string, string] = ["", "", ""]; + params[0] = fieldName || this.defaultField; + // ipv4_range(127.0.0.0, 127.255.255.255) or ipv4_range("127.0.0.0/8") or pv4_range("1.2.3.4") + if (str.length > 0 && str[0].type === "bracket" && str[0].prefix === "ipv4_range") { + const results = getValuesFromBrackets(str[0].value); + if (results.length > 0) { + params[1] = results[0]; + if (results.length > 1) { + params[2] = results[1]; + } + } + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.StringRange, + name: 'String range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Start", + type: "string", + editor: SingleCharInput, + }, { + name: "End", + type: "string", + editor: SingleCharInput, + }], + defaultParams: [this.defaultField, "", ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const start = model.params[1] as string; + const end = model.params[2] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `string_range(${start}, ${end})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string, string] = ["", "", ""]; + params[0] = fieldName || this.defaultField; + // string_range(A, C) + if (str.length > 0 && str[0].type === "bracket" && str[0].prefix === "string_range") { + const results = getValuesFromBrackets(str[0].value); + params[1] = results[0] || ""; + params[2] = results[1] || ""; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.LengthRange, + name: 'Length range', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Start", + type: "string", + editor: NumberEditor, + }, { + name: "End", + type: "string", + editor: NumberEditor, + }], + defaultParams: [this.defaultField, "5", "10"], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const start = model.params[1] as string; + const end = model.params[2] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `len_range(${start}, ${end})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: [string, string, string] = ["", "5", "10"]; + params[0] = fieldName || this.defaultField; + // len_range(5, 10) or len_range(5, inf) + if (str.length > 0 && str[0].type === "bracket" && str[0].prefix === "len_range") { + const results = getValuesFromBrackets(str[0].value); + params[1] = results[0] || "5"; + params[2] = results[1] || "10"; + str.shift(); + } + return { params, length: length - str.length }; + }, + }, + { + id: VictoriaLogsOperationId.ValueType, + name: 'Value type', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Type", + type: "string", + editor: FieldValueTypeEditor, + }], + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const type = model.params[1] as string; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `value_type(${type})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + return parseCompareOperation(str, fieldName || this.defaultField); + }, + }, + { + id: VictoriaLogsOperationId.EqField, + name: 'Equal field', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Field", + type: "string", + editor: FieldEditor, + }], + alternativesKey: "compare", + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field1 = model.params[0] as string; + const field2 = model.params[1] as string; + let expr = ""; + if (field1 !== this.defaultField) { + expr = `${quoteString(field1)}:`; + } + expr += `eq_field(${quoteString(field2)})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + return parseCompareOperation(str, fieldName || this.defaultField); + }, + }, + { + id: VictoriaLogsOperationId.LeField, + name: 'Less or equal field', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Field", + type: "string", + editor: FieldEditor, + }], + alternativesKey: "compare", + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field1 = model.params[0] as string; + const field2 = model.params[1] as string; + let expr = ""; + if (field1 !== this.defaultField) { + expr = `${quoteString(field1)}:`; + } + expr += `le_field(${quoteString(field2)})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + return parseCompareOperation(str, fieldName || this.defaultField); + }, + }, + { + id: VictoriaLogsOperationId.LtField, + name: 'Less than field', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Field", + type: "string", + editor: FieldEditor, + }], + alternativesKey: "compare", + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field1 = model.params[0] as string; + const field2 = model.params[1] as string; + let expr = ""; + if (field1 !== this.defaultField) { + expr = `${quoteString(field1)}:`; + } + expr += `lt_field(${quoteString(field2)})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + return parseCompareOperation(str, fieldName || this.defaultField); + }, + }, + { + id: VictoriaLogsOperationId.Logical, + name: 'Logical filter', + params: [{ + name: "Field", + type: "string", + editor: FieldEditor, + }, { + name: "Query", + type: "string", + editor: LogicalFilterEditor, + }], + defaultParams: [this.defaultField, ""], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Filters, + renderer: (model, def, innerExpr) => { + const field = model.params[0] as string; + const subqueryField = model.params[1] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + let expr = ""; + if (field !== this.defaultField) { + expr = `${quoteString(field)}:`; + } + expr += `(${subquery})`; + return pipeExpr(innerExpr, expr); + }, + addOperationHandler: addVictoriaOperation, + explainHandler: () => `[logical-filter](https://docs.victoriametrics.com/victorialogs/logsql/#logical-filter)`, + splitStringByParams: (str: SplitString[], fieldName?: string) => { + let length = str.length; + let params: string[] = ["", ""]; + params[0] = fieldName || this.defaultField; + if (str.length > 0) { + if (str[0].type === "bracket") { + params[1] = str[0].raw_value.slice(1, -1); + str.shift(); + } + } + return { params, length: length - str.length }; + }, + }, + // Operators + { + id: VictoriaLogsOperationId.AND, + name: 'AND', + params: [], + alternativesKey: "operators", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Operators, + renderer: (model, def, innerExpr) => innerExpr + 'AND', + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.OR, + name: 'OR', + params: [], + alternativesKey: "operators", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Operators, + renderer: (model, def, innerExpr) => innerExpr + ' OR', + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + }, + { + id: VictoriaLogsOperationId.NOT, + name: 'NOT', + params: [], + alternativesKey: "operators", + defaultParams: [], + toggleable: true, + category: VictoriaLogsQueryOperationCategory.Operators, + renderer: (model, def, innerExpr) => innerExpr + ' NOT', + addOperationHandler: addVictoriaOperation, + splitStringByParams: (str: SplitString[]) => { + return { params: [], length: 0 }; + }, + } + ]; + } +} + +function parseNumber(str: SplitString, defaultValue: undefined): number | undefined; +function parseNumber(str: SplitString, defaultValue: number): number; +function parseNumber(str: SplitString, defaultValue: number | undefined) { + let result = defaultValue; + if (str === undefined) { return result; } + if (!isValue(str)) { + return result; + } + let value = ''; + value = getValue(str); + value = value.replace(/_/g, ''); + // Regex to extract only integer or float (no suffix) + const match = value.match(/^(-?\d+(?:\.\d+)?)/); + if (!match) { return result; } + let num = parseFloat(match[1]); + if (Number.isNaN(num)) { return result; } + return num; +} + +function parseFirstLastPipe(str: SplitString[]) { + let length = str.length; + let params: [number, string, boolean, string] = [0, "", false, ""]; + do { + if (str.length >= 2) { // 10 by (request_duration) + const value = parseNumber(str[0], undefined); + if (value === undefined) { + break; + } + params[0] = value; + str.shift(); + if (str[0].type === "space" && str[0].value === "by") { + str.shift(); + } + if (str.length === 0 || str[0].type !== "bracket") { + break; + } + params[1] = str[0].raw_value.slice(1, -1); + str = str.slice(1); + if (str.length > 0 && str[0].type === "space" && str[0].value === "desc") { + params[2] = true; + str.shift(); + } + if (str.length >= 3) { // partition by (host) + if (str[0].type === "space" && str[0].value === "partition") { + if (str[1].type === "space" && str[1].value === "by") { + if (str[2].type === "bracket") { + params[3] = str[2].raw_value.slice(1, -1); + str = str.slice(3); + } + } + } + } + } + } while (false); + return { params, length: length - str.length }; +} + +function parseStatsPipe(str: SplitString[]) { + const params: string[] = [""]; + const length = str.length; + if (str.length > 0) { // check for stats by fields + if (str[0].type === "space" && str[0].value === "by") { // optional + str.shift(); + } + if (str[0].type === "bracket" && str[0].prefix === "") { + params[0] = str[0].raw_value.slice(1, -1); + str = str.slice(1); + } + } + params[1] = buildSplitString(str); + return { params, length: length }; // everything of str +} + +function renderStatsPipe(model: QueryBuilderOperation, def: QueryBuilderOperationDefinition, innerExpr: string): string { + const statsBy = model.params[0] as string; + const subqueryField = model.params[1] as unknown; + const subquery = (typeof subqueryField === "string") ? subqueryField : (subqueryField as { expr: string }).expr; + let expr = model.id; + if (statsBy !== "") { + expr += ` by (${statsBy})`; + } + expr += " " + subquery; + return pipeExpr(innerExpr, expr); +} + +function parseLenPipe(str: SplitString[], fnName: string) { + const length = str.length; + const params: [string, string] = ["", ""]; + if (str.length >= 2) { + let value; + if (str[0].type === "bracket" && str[0].prefix === fnName) { + value = str[0].value; + str.shift(); + } else if (str[0].type === "space" && str[0].value === fnName && str[1].type === "bracket") { + value = str[1].value; + str = str.slice(2); + } else { + return { params, length: length - str.length }; + } + if (value.length > 0) { + if (value[0].type === "space") { + params[0] = value[0].value; + } else if (value[0].type === "quote") { + params[0] = unquoteString(value[0].value); + } + } + if (str[0].type === "space" && str[0].value === "as") { + str = str.slice(1); + if (str.length > 0) { + if (isValue(str[0])) { + params[1] = getValue(str[0]); + str.shift(); + } + } + } + } + return { params, length: length - str.length }; +} + +function getFieldValue(str: SplitString[], defaultValue = ""): string { + if (str.length === 0) { + return defaultValue; + } + let value = defaultValue; + if (isValue(str[0])) { + value = getValue(str[0]); + if (str[0].value === ",") { + value = defaultValue; + } else { + str.shift(); + } + } + return value; +} + +function renderStatsOperation(hasLimit = false, sortBy = false) { + function renderStatsOperation(model: QueryBuilderOperation, def: QueryBuilderOperationDefinition, innerExpr: string): string { + const fields = model.params[0] as string; + const operation = model.id; + let expr = `${operation}(${fields})`; + let offset = 1; + if (sortBy) { + const sortByFields = model.params[offset++] as string; + if (sortByFields !== "") { + expr += ` sort by (${sortByFields})`; + } + } + if (hasLimit) { + const Limit = model.params[offset++] as number; + if (Limit > 0) { + expr += ` limit ${Limit}`; + } + } + const resultField = model.params[offset++] as string; + const condition = model.params[offset] as string; + if (condition !== "") { + expr += ` if (${condition})`; + } + if (resultField !== "") { + expr += ` ${quoteString(resultField)}`; + } + return pipeExpr(innerExpr, expr); + } + return renderStatsOperation; +} + +type StatsParamsWithoutLimit = [field: string, resultField: string, condition: string]; +type StatsParamsWithLimit = [field: string, limit: number, resultField: string, condition: string]; +type StatsParamsWithoutLimitWithSort = [field: string, sortBy: string, resultField: string, condition: string]; +type StatsParamsWithLimitWithSort = [field: string, sortBy: string, limit: number, resultField: string, condition: string]; + +type StatsParamsParseFnWithLimit = (str: SplitString[]) => { params: StatsParamsWithLimit, length: number }; +type StatsParamsParseFnWithoutLimit = (str: SplitString[]) => { params: StatsParamsWithoutLimit, length: number }; +type StatsParamsParseFnWithLimitWithSort = (str: SplitString[]) => { params: StatsParamsWithLimitWithSort, length: number }; +type StatsParamsParseFnWithoutLimitWithSort = (str: SplitString[]) => { params: StatsParamsWithoutLimitWithSort, length: number }; + +function parseStatsOperationWithLimit(str: SplitString[], sortBy: true): { params: StatsParamsWithLimitWithSort, length: number } +function parseStatsOperationWithLimit(str: SplitString[], sortBy: false): { params: StatsParamsWithLimit, length: number } +function parseStatsOperationWithLimit(str: SplitString[], sortBy = false) { + let params: StatsParamsWithLimitWithSort | StatsParamsWithLimit; + if (sortBy) { + params = ["", "", 0, "", ""]; + } else { + params = ["", 0, "", ""]; + } + const length = str.length; + if (str.length > 0) { + if (str[0].type === "space") { + str.shift(); + } + if (str[0].type === "bracket") { + params[0] = str[0].raw_value.slice(1, -1); + str.shift(); + } + let offset = 1; + if (sortBy && str.length > 0) { + if (str[0].type === "space" && str[0].value === "sort") { + str = str.slice(1); + if (str.length > 0 && str[0].type === "space" && str[0].value === "by") { + str = str.slice(1); + if (str.length > 0 && str[0].type === "bracket") { + params[offset++] = buildSplitString(str[0].value); + str.shift(); + } + } + } + } + if (str.length > 0) { + if (str[0].type === "space" && str[0].value === "limit") { + str.shift(); + if (str.length > 0) { + params[offset++] = parseNumber(str[0], 0); + str.shift(); + } + } + } + params[offset + 1] = getConditionFromString(str); + if (str.length > 0 && str[0].value === "as") { + str.shift(); + } + params[offset] = getFieldValue(str); + } + return { params, length: length - str.length }; +} + +function parseStatsOperationWithoutLimit(str: SplitString[], sortBy: false): { params: StatsParamsWithoutLimitWithSort, length: number } +function parseStatsOperationWithoutLimit(str: SplitString[], sortBy: true): { params: StatsParamsWithoutLimit, length: number } +function parseStatsOperationWithoutLimit(str: SplitString[], sortBy: boolean) { + let params: StatsParamsWithoutLimit | StatsParamsWithoutLimitWithSort; + if (sortBy) { + params = ["", "", "", ""]; + } else { + params = ["", "", ""]; + } + const length = str.length; + if (str.length > 0) { + if (str[0].type === "space") { + str.shift(); + } + if (str[0].type === "bracket") { + params[0] = str[0].raw_value.slice(1, -1); + str.shift(); + } + let offset = 1; + if (sortBy && str.length > 0) { + if (str[0].type === "space" && str[0].value === "sort") { + str = str.slice(1); + if (str.length > 0 && str[0].type === "space" && str[0].value === "by") { + str = str.slice(1); + if (str.length > 0 && str[0].type === "bracket") { + params[offset++] = buildSplitString(str[0].value); + str.shift(); + } + } + } + } + params[offset + 1] = getConditionFromString(str); + if (str.length > 0 && str[0].value === "as") { + str.shift(); + } + params[offset] = getFieldValue(str); + } + return { params, length: length - str.length }; +} + +function parseStatsOperation(hasLimit: true, sortBy: true): StatsParamsParseFnWithLimitWithSort; +function parseStatsOperation(hasLimit: false, sortBy: true): StatsParamsParseFnWithoutLimitWithSort; +function parseStatsOperation(hasLimit: true, sortBy: false): StatsParamsParseFnWithLimit; +function parseStatsOperation(hasLimit: false, sortBy: false): StatsParamsParseFnWithoutLimit; +function parseStatsOperation(hasLimit: true): StatsParamsParseFnWithLimit; +function parseStatsOperation(hasLimit: false): StatsParamsParseFnWithoutLimit; + +function parseStatsOperation(hasLimit: boolean, sortBy = false): StatsParamsParseFnWithLimitWithSort | StatsParamsParseFnWithoutLimitWithSort | StatsParamsParseFnWithLimit | StatsParamsParseFnWithoutLimit { + if (hasLimit && sortBy === true) { + return (str: SplitString[]) => parseStatsOperationWithLimit(str, true); + } + if (hasLimit && sortBy === false) { + return (str: SplitString[]) => parseStatsOperationWithLimit(str, false); + } + if (!hasLimit && sortBy === true) { + return (str: SplitString[]) => parseStatsOperationWithoutLimit(str, true); + } + // !hasLimit && sortBy === false + return (str: SplitString[]) => parseStatsOperationWithoutLimit(str, false); +} + +function getFieldList(str: SplitString[]) { + let fields: string[] = []; + while (str.length > 0) { + if (str[0].type === "space" || str[0].type === "quote") { + fields.push(str[0].value); + str.shift(); + if (str.length > 0 && str[0].value === ",") { + str.shift(); + continue; + } + } + break; + } + return fields; +} + +function parseCompareOperation(str: SplitString[], fieldName: string) { + let length = str.length; + let params: string[] = [fieldName, ""]; + const compareOps = ["value_type", "eq_field", "le_field", "lt_field"]; + if (str.length > 0 && str[0].type === "bracket" && compareOps.includes(str[0].prefix)) { + params[1] = getValuesFromBrackets(str[0].value)[0] || ""; + str.shift(); + } + return { params, length: length - str.length }; +} + +function parseExtractOperation(str: SplitString[], defaultField: string) { + let length = str.length; + let params: [string, string, boolean, boolean, string] = [defaultField, "", false, false, ""]; + params[4] = getConditionFromString(str); + if (str.length === 0 || str[0].type !== "quote") { + return { params, length: 0 }; + } + // "ip= " from _msg + params[1] = unquoteString(str[0].value); + str = str.slice(1); + if (str.length >= 2) { + if (str[0].type === "space" && str[0].value === "from") { + str = str.slice(1); + if (isValue(str[0])) { + params[0] = getValue(str[0]); + str.shift(); + } + } + } + let i = 0; + while (str.length > 0) { + if (i >= 2) { + break; + } else if (str[0].type === "space") { + if (str[0].value === "keep_original_fields") { + str = str.slice(1); + params[2] = true; + } else if (str[0].value === "skip_empty_results") { + str = str.slice(1); + params[3] = true; + } else { + break; + } + } else { + break; + } + i++; + } + return { params, length: length - str.length }; +} + +function buildExtractOperation(model: QueryBuilderOperation, innerExpr: string, defaultField: string): string { + const modelId = model.id; + const fromField = model.params[0] as string; + const pattern = model.params[1] as string; + const keepOriginalFields = model.params[2] as boolean; + const skipEmptyResults = model.params[3] as boolean; + const condition = model.params[4] as string; + let expr = modelId; + if (condition !== "") { + expr += ` if (${condition})`; + } + expr += " " + quoteString(pattern, true); + if (fromField !== defaultField) { + expr += ` from ${quoteString(fromField)}`; + } + if (keepOriginalFields) { + expr += " keep_original_fields"; + } + if (skipEmptyResults) { + expr += " skip_empty_results"; + } + return pipeExpr(innerExpr, expr); +} + +function parsePrefixFieldList(str: SplitString[]): string[] { + let fields: string[] = []; + while (str.length > 0) { + if (str[0].type === "space" || str[0].type === "quote") { + let value = str[0].value; + if (str.length > 1 && str[0].type === "quote") { + if (str[1].value === "*") { + str.shift(); + value += "*"; + } + } + str.shift(); + fields.push(value); + if (str.length > 0 && str[0].value === ",") { + str.shift(); + continue; + } + } + break; + } + return fields; +} diff --git a/src/components/QueryEditor/QueryBuilder/QueryBuilder.tsx b/src/components/QueryEditor/QueryBuilder/QueryBuilder.tsx index a1f4810a..807bfd8b 100644 --- a/src/components/QueryEditor/QueryBuilder/QueryBuilder.tsx +++ b/src/components/QueryEditor/QueryBuilder/QueryBuilder.tsx @@ -1,16 +1,15 @@ + import { css } from "@emotion/css"; -import React, { Fragment, memo } from 'react'; +import React, { memo, useMemo } from 'react'; -import { GrafanaTheme2, TimeRange } from "@grafana/data"; +import { GrafanaTheme2, TimeRange, DataSourceApi } from "@grafana/data"; +import { OperationList } from '@grafana/plugin-ui'; import { useStyles2 } from "@grafana/ui"; import { VictoriaLogsDatasource } from "../../../datasource"; -import { FilterVisualQuery, VisualQuery } from "../../../types"; +import { VisualQuery } from "../../../types"; -import QueryBuilderAddFilter from "./components/QueryBuilderAddFilter"; -import QueryBuilderFieldFilter from "./components/QueryBuilderFilters/QueryBuilderFieldFilter"; -import QueryBuilderSelectOperator from "./components/QueryBuilderOperators/QueryBuilderSelectOperator"; -import { DEFAULT_FILTER_OPERATOR } from "./utils/parseToString"; +import { QueryModeller } from "./QueryModellerClass"; interface Props { query: VisualQuery; @@ -20,88 +19,30 @@ interface Props { onRunQuery: () => void; } -const QueryBuilder = memo(({ datasource, query, onChange, timeRange }) => { +const QueryBuilder = memo(({ datasource, query, onChange, onRunQuery, timeRange }) => { const styles = useStyles2(getStyles); - const { filters } = query + const queryModeller = useMemo(() => { + return new QueryModeller([]); + }, []); + const onVisQueryChange = (visQuery: VisualQuery) => { + const expr = queryModeller.renderQuery(visQuery); + onChange({ ...visQuery, expr: expr }); + }; return (
-
) }); -interface QueryBuilderFilterProps { - datasource: VictoriaLogsDatasource; - query: VisualQuery; - filters: FilterVisualQuery; - indexPath: number[]; - timeRange?: TimeRange; - onChange: (query: VisualQuery) => void; -} - -const QueryBuilderFilter = (props: QueryBuilderFilterProps) => { - const styles = useStyles2(getStyles); - const { datasource, filters, query, indexPath, timeRange, onChange } = props - const isRoot = !indexPath.length - return ( -
- {filters.values.map((filter, index) => ( - -
- {typeof filter === 'string' - ? - : - } -
- {index !== filters.values.length - 1 && ( - - )} - - ) - )} - {/* for new filters*/} - {!filters.values.length && ( - - )} - -
- ) -} - const getStyles = (theme: GrafanaTheme2) => { return { builderWrapper: css` @@ -110,21 +51,6 @@ const getStyles = (theme: GrafanaTheme2) => { align-items: center; gap: ${theme.spacing(1)}; `, - filterWrapper: css` - display: flex; - flex-wrap: wrap; - align-items: center; - gap: ${theme.spacing(1)}; - border: 1px solid ${theme.colors.border.strong}; - background-color: ${theme.colors.border.weak}; - padding: ${theme.spacing(1)}; - `, - filterItem: css` - display: flex; - align-items: center; - justify-content: flex-start; - gap: ${theme.spacing(1)}; - ` }; }; diff --git a/src/components/QueryEditor/QueryBuilder/QueryBuilderContainer.tsx b/src/components/QueryEditor/QueryBuilder/QueryBuilderContainer.tsx index c10a271f..8ad32392 100644 --- a/src/components/QueryEditor/QueryBuilder/QueryBuilderContainer.tsx +++ b/src/components/QueryEditor/QueryBuilder/QueryBuilderContainer.tsx @@ -1,41 +1,41 @@ import { css } from "@emotion/css"; -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { GrafanaTheme2, TimeRange } from '@grafana/data'; import { useStyles2 } from '@grafana/ui'; import { VictoriaLogsDatasource } from "../../../datasource"; -import { Query, VisualQuery } from "../../../types"; +import { VisualQuery } from "../../../types"; import QueryBuilder from "./QueryBuilder"; -import { buildVisualQueryFromString } from "./utils/parseFromString"; -import { parseVisualQueryToString } from "./utils/parseToString"; +import { parseExprToVisualQuery } from "./QueryModeller"; - -export interface Props { - query: Query; +export interface Props { + query: Q; datasource: VictoriaLogsDatasource; - onChange: (update: Query) => void; + onChange: (update: Q) => void; onRunQuery: () => void; timeRange?: TimeRange; } -export function QueryBuilderContainer(props: Props) { +export function QueryBuilderContainer(props: Props) { const styles = useStyles2(getStyles); - const { query, onChange, onRunQuery, datasource, timeRange } = props + const { query, onChange, onRunQuery, datasource, timeRange } = props; + + const visQuery = useMemo(() => { + return parseExprToVisualQuery(query.expr).query; + }, [query.expr]); - const [state, setState] = useState<{expr: string, visQuery: VisualQuery}>({ + const [state, setState] = useState<{ expr: string, visQuery: VisualQuery }>({ expr: query.expr, - visQuery: buildVisualQueryFromString(query.expr).query + visQuery: visQuery, }) const onVisQueryChange = (visQuery: VisualQuery) => { - const expr = parseVisualQueryToString(visQuery); - setState({ expr, visQuery }) - onChange({ ...props.query, expr: expr }); + setState({ expr: visQuery.expr, visQuery }) + onChange({ ...props.query, expr: visQuery.expr }); }; - return ( <> -
+

- {query.expr !== '' && query.expr} + {state.expr}

); diff --git a/src/components/QueryEditor/QueryBuilder/QueryEditorModeToggle.tsx b/src/components/QueryEditor/QueryBuilder/QueryEditorModeToggle.tsx index 4f52c713..f7d80521 100644 --- a/src/components/QueryEditor/QueryBuilder/QueryEditorModeToggle.tsx +++ b/src/components/QueryEditor/QueryBuilder/QueryEditorModeToggle.tsx @@ -10,7 +10,7 @@ export interface Props { } const editorModes = [ - { label: 'Beta Builder', value: QueryEditorMode.Builder }, + { label: 'Builder', value: QueryEditorMode.Builder }, { label: 'Code', value: QueryEditorMode.Code }, ]; diff --git a/src/components/QueryEditor/QueryBuilder/QueryModeller.tsx b/src/components/QueryEditor/QueryBuilder/QueryModeller.tsx new file mode 100644 index 00000000..70b4b0e8 --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/QueryModeller.tsx @@ -0,0 +1,98 @@ +import { QueryBuilderOperation } from "@grafana/plugin-ui"; + +import { VisualQuery } from "../../../types"; + +import { OperationDefinitions } from "./Operations"; +import { QueryModeller } from './QueryModellerClass'; +import { VictoriaLogsQueryOperationCategory } from './VictoriaLogsQueryOperationCategory'; +import { parseOperation, parseStatsOperation } from './utils/operationParser'; +import { splitByOperator, splitByUnescapedPipe, splitString } from './utils/stringSplitter'; + +export function createQueryModellerWithDefaultField(defaultField: string, categories: VictoriaLogsQueryOperationCategory[]) { + const queryModeller = new QueryModeller(new OperationDefinitions(defaultField).all(), categories); + return queryModeller; +} + +export function createQueryModellerForCategories(categories: VictoriaLogsQueryOperationCategory[] = Object.values(VictoriaLogsQueryOperationCategory)) { + const queryModeller = new QueryModeller(new OperationDefinitions().all(), categories); + return queryModeller; +} + +export const buildVisualQueryToString = (query: VisualQuery, queryModeller?: QueryModeller): string => { + if (!queryModeller) { + queryModeller = createQueryModellerForCategories(Object.values(VictoriaLogsQueryOperationCategory)); + } + return queryModeller.renderQuery(query); +} + +const handleExpression = (expr: string, defaultField = "_msg", queryModeller?: QueryModeller): QueryBuilderOperation[] => { + let operationList: QueryBuilderOperation[] = []; + // first split by pipes then by operators + let lastOpWasOperator = false; + const fullSplitString = splitString(expr || ""); + let operationQueryModeller = queryModeller; + if (defaultField !== "_msg" || queryModeller === undefined) { + operationQueryModeller = createQueryModellerWithDefaultField(defaultField, Object.values(VictoriaLogsQueryOperationCategory)) + } else { + operationQueryModeller = queryModeller; + } + for (const splitByPipes of splitByUnescapedPipe(fullSplitString)) { + for (let splitByOp of splitByOperator(splitByPipes)) { + if (splitByOp.length > 0 && splitByOp[0].type === "space") { + if (["and", "or", "not"].includes(splitByOp[0].value.toLowerCase())) { + operationList.push({ + id: splitByOp[0].value.toLowerCase(), + params: [], + }) + lastOpWasOperator = true; + continue; + } + } + const comments = splitByOp.filter(part => part.type === "comment"); + if (comments.length > 0) { + splitByOp = splitByOp.filter(part => part.type !== "comment"); + } + while (splitByOp.length > 0) { + const parsedOperation = parseOperation(splitByOp, lastOpWasOperator, operationQueryModeller); + if (parsedOperation) { + operationList.push(parsedOperation.operation); + splitByOp = splitByOp.slice(parsedOperation.length); + if (operationQueryModeller.getOperationDefinition(parsedOperation.operation.id).category === VictoriaLogsQueryOperationCategory.Stats) { + while (splitByOp.length > 0 && splitByOp[0].value === ",") { + splitByOp = splitByOp.slice(1); + let statsOperation = parseStatsOperation(splitByOp, operationQueryModeller); + if (statsOperation) { + operationList.push(statsOperation.operation); + splitByOp = splitByOp.slice(statsOperation.length); + } + } + } + } else { + break; + } + } + while (comments.length > 0) { + const comment = comments.shift(); + if (comment) { + operationList.push({ + id: "comment", + params: [comment.value], + }); + } + } + lastOpWasOperator = false; + } + } + return operationList; +} + +export const parseExprToVisualQuery = (expr: string, defaultField = "_msg", queryModeller?: QueryModeller): { query: VisualQuery, errors: string[] } => { + const newOperations = handleExpression(expr, defaultField, queryModeller); + const query: VisualQuery = { + labels: [], + operations: newOperations, + expr: expr, + }; + let errors: string[] = []; + return { query, errors }; +} diff --git a/src/components/QueryEditor/QueryBuilder/QueryModellerClass.tsx b/src/components/QueryEditor/QueryBuilder/QueryModellerClass.tsx new file mode 100644 index 00000000..fe06248b --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/QueryModellerClass.tsx @@ -0,0 +1,126 @@ +import { QueryBuilderLabelFilter, QueryBuilderOperation, VisualQueryModeller } from "@grafana/plugin-ui"; + +import { VisualQuery } from "../../../types"; + +import { VictoriaLogsOperationId, VictoriaQueryBuilderOperationDefinition, OperationDefinitions } from "./Operations"; +import { VictoriaLogsQueryOperationCategory } from "./VictoriaLogsQueryOperationCategory"; + +declare abstract class VictoriaVisualQueryModeller implements VisualQueryModeller { + innerQueryPlaceholder: string; + constructor(operationDefinitions: VictoriaQueryBuilderOperationDefinition[], innerQueryPlaceholder?: string); + abstract renderOperations(queryString: string, operations: QueryBuilderOperation[]): string; + abstract renderLabels(labels: QueryBuilderLabelFilter[]): string; + abstract renderQuery(query: VisualQuery, nested?: boolean): string; + getOperationsForCategory(category: string): VictoriaQueryBuilderOperationDefinition[]; + getAlternativeOperations(key: string): VictoriaQueryBuilderOperationDefinition[]; + getCategories(): string[]; + getOperationDefinition(id: string): VictoriaQueryBuilderOperationDefinition | undefined; +} + +export class QueryModeller implements VictoriaVisualQueryModeller { + innerQueryPlaceholder = ' '; + operationDefinitions: VictoriaQueryBuilderOperationDefinition[]; + mappedDefinitions: Record; + categories: string[]; + onlyStats: boolean; + constructor(operationDefinitions: VictoriaQueryBuilderOperationDefinition[], categories: string[] = Object.values(VictoriaLogsQueryOperationCategory)) { + if (operationDefinitions.length === 0) { + operationDefinitions = new OperationDefinitions().all(); + } + this.onlyStats = categories.includes(VictoriaLogsQueryOperationCategory.Stats) && categories.length === 1; + this.operationDefinitions = operationDefinitions.filter(op => { + return categories.some(category => op.category === category); + }); + this.categories = this.operationDefinitions.reduce( + (acc, operation: VictoriaQueryBuilderOperationDefinition) => { + if (!acc.includes(operation.category)) { + acc.push(operation.category); + } + return acc; + }, + [] as string[] + ); + this.mappedDefinitions = this.operationDefinitions.reduce( + (acc, operation: VictoriaQueryBuilderOperationDefinition) => { + acc[operation.id as VictoriaLogsOperationId] = operation; + return acc; + }, + {} as Record + ); + } + + getOperationsForCategory(category: string): VictoriaQueryBuilderOperationDefinition[] { + return this.operationDefinitions.filter((operation: VictoriaQueryBuilderOperationDefinition) => { + return operation.category === category; + }); + } + + getAlternativeOperations(key: string): VictoriaQueryBuilderOperationDefinition[] { + if (key === undefined) { + return []; + } + return this.operationDefinitions.filter((operation: VictoriaQueryBuilderOperationDefinition) => { + return operation.alternativesKey === key; + }); + } + + getCategories(): string[] { + return this.categories; + } + + getOperationDefinition(id: string) { + return this.mappedDefinitions[id as VictoriaLogsOperationId]; + } + + renderOperations(queryString: string, operations: QueryBuilderOperation[]): string { + let opDefs = []; + for (let i = 0; i < operations.length; i++) { + const operation = operations[i]; + if (operation.disabled) { + continue; + } + const operationDef = this.getOperationDefinition(operation.id); + opDefs.push(operationDef); + if (!operationDef) { + continue; + } + + if (this.onlyStats) { + if (queryString !== "") { + queryString += ", "; + } + queryString += operationDef.renderer(operation, operationDef, ""); + } else if (i > 0 && checkIsFilter(opDefs[i - 1]) && checkIsFilter(operationDef)) { + queryString += " " + operationDef.renderer(operation, operationDef, ""); + } else if (operationDef.id === VictoriaLogsOperationId.Comment) { + queryString += operationDef.renderer(operation, operationDef, ""); + } else if (operationDef.category === VictoriaLogsQueryOperationCategory.Operators) { + if (i > 0 && !checkIsFilter(opDefs[i - 1])) { + queryString += " |" + operationDef.renderer(operation, operationDef, ""); + } + } else { + queryString = operationDef.renderer(operation, operationDef, queryString); + } + } + return queryString.trim(); + } + + renderQuery(query: { operations: QueryBuilderOperation[] }, nested?: boolean): string { + const queryString = this.renderOperations("", query.operations); + return queryString; + } + + renderLabels(labels: QueryBuilderLabelFilter[]): string { + return ''; + } +} + +function checkIsFilter(def: VictoriaQueryBuilderOperationDefinition | undefined): boolean { + if (!def) { + return false; + } + if (def.category === VictoriaLogsQueryOperationCategory.Filters || def.category === VictoriaLogsQueryOperationCategory.Operators) { + return true; + } + return def.id === VictoriaLogsOperationId.FieldContainsAnyValueFromVariable; +} diff --git a/src/components/QueryEditor/QueryBuilder/VictoriaLogsQueryOperationCategory.ts b/src/components/QueryEditor/QueryBuilder/VictoriaLogsQueryOperationCategory.ts new file mode 100644 index 00000000..1fac43be --- /dev/null +++ b/src/components/QueryEditor/QueryBuilder/VictoriaLogsQueryOperationCategory.ts @@ -0,0 +1,8 @@ + +export enum VictoriaLogsQueryOperationCategory { + Filters = 'Filters', + Operators = 'Operators', + Pipes = 'Pipes', + Stats = 'Stats', + Special = 'Special', +} diff --git a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderAddFilter.tsx b/src/components/QueryEditor/QueryBuilder/components/QueryBuilderAddFilter.tsx deleted file mode 100644 index 8ae8acd7..00000000 --- a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderAddFilter.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import { css } from "@emotion/css"; -import React, { useCallback } from 'react'; - -import { GrafanaTheme2 } from "@grafana/data"; -import { Button, useStyles2 } from "@grafana/ui"; - -import { VisualQuery } from "../../../../types"; -import { DEFAULT_FILTER_OPERATOR } from "../utils/parseToString"; - -interface Props { - query: VisualQuery; - onAddFilter: (query: VisualQuery) => void; -} - -const QueryBuilderAddFilter = ({ query, onAddFilter }: Props) => { - const styles = useStyles2(getStyles); - - const handleAddFilter = useCallback(() => { - onAddFilter({ - ...query, filters: { - ...query.filters, - values: [...query.filters.values, ''], - operators: [...query.filters.operators, DEFAULT_FILTER_OPERATOR] - } - }) - }, [onAddFilter, query]) - - return ( -
- -
- ) -} - -const getStyles = (_theme: GrafanaTheme2) => { - return { - wrapper: css` - align-self: flex-end; - display: flex; - align-items: center; - justify-content: center; - `, - }; -}; - -export default QueryBuilderAddFilter; diff --git a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderFilters/QueryBuilderFieldFilter.tsx b/src/components/QueryEditor/QueryBuilder/components/QueryBuilderFilters/QueryBuilderFieldFilter.tsx deleted file mode 100644 index c941c212..00000000 --- a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderFilters/QueryBuilderFieldFilter.tsx +++ /dev/null @@ -1,152 +0,0 @@ -import { css } from "@emotion/css"; -import React, { useMemo, useState } from "react"; - -import { GrafanaTheme2, SelectableValue, TimeRange } from "@grafana/data"; -import { IconButton, Label, Select, useStyles2 } from "@grafana/ui"; - -import { VictoriaLogsDatasource } from "../../../../../datasource"; -import { escapeLabelValueInExactSelector } from "../../../../../languageUtils"; -import { FilterFieldType, VisualQuery } from "../../../../../types"; -import { deleteByIndexPath } from "../../utils/modifyFilterVisualQuery/deleteByIndexPath"; -import { updateValueByIndexPath } from "../../utils/modifyFilterVisualQuery/updateByIndexPath"; -import { DEFAULT_FIELD, filterVisualQueryToString } from "../../utils/parseToString"; - -interface Props { - datasource: VictoriaLogsDatasource; - filter: string; - query: VisualQuery; - indexPath: number[]; - timeRange?: TimeRange; - onChange: (query: VisualQuery) => void; -} - -const QueryBuilderFieldFilter = ({ datasource, filter, query, indexPath, timeRange, onChange }: Props) => { - const styles = useStyles2(getStyles); - - const [fieldNames, setFieldNames] = useState[]>([]) - const [isLoadingFieldNames, setIsLoadingFieldNames] = useState(false) - - const [fieldValues, setFieldValues] = useState[]>([]) - const [isLoadingFieldValues, setIsLoadingFieldValues] = useState(false) - - const { field, fieldValue } = useMemo(() => { - const regex = /("[^"]*"|'[^']*'|\S+)\s*:\s*("[^"]*"|'[^']*'|\S+)?|\S+/i - const matches = filter.match(regex); - if (!matches || matches.length < 1) { - return {}; - } - const field = matches[1] || DEFAULT_FIELD - const fieldValue = matches[2] ?? (matches[1] ? "" : matches[0]) - - return { field, fieldValue } - }, [filter]) - - const handleRemoveFilter = () => { - onChange({ - ...query, - filters: deleteByIndexPath(query.filters, indexPath) - }) - } - - const handleSelect = (type: FilterFieldType) => ({ value: selected }: SelectableValue) => { - const fullFilter = type === FilterFieldType.FieldName - ? `${selected}: ${fieldValue || ''}` - : `${field || ''}: ${field === '_stream' ? selected : `"${escapeLabelValueInExactSelector(selected || "")}"`} ` - - onChange({ - ...query, - filters: updateValueByIndexPath(query.filters, indexPath, fullFilter) - }) - } - - const handleCreate = (type: FilterFieldType) => (customValue: string) => { - handleSelect(type)({ value: customValue }) - } - - const handleOpenMenu = (type: FilterFieldType) => async () => { - const setterLoading = type === FilterFieldType.FieldName ? setIsLoadingFieldNames : setIsLoadingFieldValues - const setterValues = type === FilterFieldType.FieldName ? setFieldNames : setFieldValues - - setterLoading(true) - const limit = datasource.getQueryBuilderLimits(type) - const filtersWithoutCurrent = deleteByIndexPath(query.filters, indexPath) - const currentOperator = query.filters.operators[indexPath[0] - 1] || "AND" - const filters = currentOperator === "AND" ? filterVisualQueryToString(filtersWithoutCurrent, true) : "" - const list = await datasource.languageProvider?.getFieldList({ type, timeRange, field, limit, query: filters }); - const result = list ? list.map(({ value, hits }) => ({ - value, - label: value || " ", - description: `hits: ${hits}`, - })) : [] - setterValues(result) - setterLoading(false) - } - - return ( -
-
- - -
-
- -
-
- ) -} - -const getStyles = (theme: GrafanaTheme2) => { - return { - wrapper: css` - display: grid; - gap: ${theme.spacing(0.5)}; - width: max-content; - border: 1px solid ${theme.colors.border.strong}; - background-color: ${theme.colors.background.secondary}; - padding: ${theme.spacing(1)}; - `, - header: css` - display: flex; - align-items: center; - justify-content: space-between; - `, - content: css` - display: flex; - align-items: center; - justify-content: center; - gap: ${theme.spacing(0.5)}; - `, - }; -}; - -export default QueryBuilderFieldFilter diff --git a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderOperators/QueryBuilderSelectOperator.tsx b/src/components/QueryEditor/QueryBuilder/components/QueryBuilderOperators/QueryBuilderSelectOperator.tsx deleted file mode 100644 index f58471f6..00000000 --- a/src/components/QueryEditor/QueryBuilder/components/QueryBuilderOperators/QueryBuilderSelectOperator.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import { css } from "@emotion/css"; -import React from 'react'; - -import { GrafanaTheme2, SelectableValue } from "@grafana/data"; -import { Label, Select, useStyles2 } from "@grafana/ui"; - -import { VisualQuery } from "../../../../../types"; -import { updateOperatorByIndexPath } from "../../utils/modifyFilterVisualQuery/updateByIndexPath"; -import { DEFAULT_FILTER_OPERATOR } from "../../utils/parseToString"; -import { BUILDER_OPERATORS } from "../../utils/parsing"; - -interface Props { - query: VisualQuery; - operator: string; - indexPath: number[]; - onChange: (query: VisualQuery) => void; -} - -const QueryBuilderSelectOperator: React.FC = ({ query, operator, indexPath, onChange }) => { - const styles = useStyles2(getStyles); - - const handleOperatorChange = ({ value }: SelectableValue) => { - onChange({ - ...query, - filters: updateOperatorByIndexPath(query.filters, indexPath, value || DEFAULT_FILTER_OPERATOR) - }) - } - - const handleCreateOption = (customValue: string) => { - handleOperatorChange({ value: customValue }); - } - - const options = BUILDER_OPERATORS.map((op) => ({ label: op, value: op })) - - return ( -
- -