diff --git a/.github/workflows/pr-test-build.yaml b/.github/workflows/pr-test-build.yaml index 67a43e1f..2dbfb8b9 100644 --- a/.github/workflows/pr-test-build.yaml +++ b/.github/workflows/pr-test-build.yaml @@ -124,19 +124,21 @@ jobs: ? `**Source:** Fork (\`${{ needs.check-permissions.outputs.pr-repo }}\`)` : `**Source:** Branch (\`${{ needs.check-permissions.outputs.pr-ref }}\`)`; + const comment = ` + ## 🚀 Build Started + + **PR:** #${{ needs.check-permissions.outputs.pr-number }} + ${sourceInfo} + **Triggered by:** @${{ github.actor }} + + [⏳ View build progress](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + `; + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: ${{ needs.check-permissions.outputs.pr-number }}, - body: ` - ## 🚀 Build Started - - **PR:** #${{ needs.check-permissions.outputs.pr-number }} - ${sourceInfo} - **Triggered by:** @${{ github.actor }} - - [⏳ View build progress](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - ` + body: comment }); - name: Check out PR code diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a57da82..6a090168 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ * BUGFIX: fix setting `extra_stream_filters` param to `Custom query parameters`. See [this comment](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/405#issuecomment-3420418177). * BUGFIX: add `sort by (_time) asc/desc` pipe if logs are sorted in asc/desc order. In versions of grafana below `12.x.x`, you need to manually run the query if the sorting has been changed. See [#379](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/379). -* fix the issue of overwriting the `level` label. Set the calculated level into a `detected_level` label, which is supported only in Grafana version 11.0.8 and above. +* BUGFIX: fix the issue of overwriting the `level` label. Set the calculated level into a `detected_level` label, which is supported only in Grafana version 11.0.8 and above. See [#425](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/425). +* BUGFIX: fix interpolation of multi-value query variables by mapping `filterName:$filterVar` and `filterName:=$filterVar` to the `filerName:in("v1", ..., "vN")`. Support negative operators and stream tags(`{tag = $var}` to `{tag in($var)}`). Disallow interpolation in regexp with variables (e.g., `field:~$var`). See [#238](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/238). ## v0.21.1 diff --git a/src/LogsQL/regExpOperator.test.ts b/src/LogsQL/regExpOperator.test.ts new file mode 100644 index 00000000..adc52962 --- /dev/null +++ b/src/LogsQL/regExpOperator.test.ts @@ -0,0 +1,83 @@ +import { getQueryExprVariableRegExp } from './regExpOperator'; + +describe('getQueryExprVariableRegExp', () => { + it('should fing fieldName:~$var', () => { + const result = getQueryExprVariableRegExp(' fieldName:~$var '); + expect(result?.[0]).toEqual(' fieldName:~$var'); + }); + + it('should find fieldName:~$var without spaces', () => { + const result = getQueryExprVariableRegExp('fieldName:~$var'); + expect(result?.[0]).toEqual('fieldName:~$var'); + }); + + it('should find fieldName:~$var at the start of string', () => { + const result = getQueryExprVariableRegExp('fieldName:~$var and more'); + expect(result?.[0]).toEqual('fieldName:~$var'); + }); + + it('should find fieldName:~$var at the end of string', () => { + const result = getQueryExprVariableRegExp('some query fieldName:~$var'); + expect(result?.[0]).toEqual(' fieldName:~$var'); + }); + + it('should find variable with underscores', () => { + const result = getQueryExprVariableRegExp('fieldName:~$my_var_name'); + expect(result?.[0]).toEqual('fieldName:~$my_var_name'); + }); + + it('should find variable with numbers', () => { + const result = getQueryExprVariableRegExp('fieldName:~$var123'); + expect(result?.[0]).toEqual('fieldName:~$var123'); + }); + + it('should find complex field name with variable', () => { + const result = getQueryExprVariableRegExp('complex_field_name:~$variable'); + expect(result?.[0]).toEqual('complex_field_name:~$variable'); + }); + + it('should return null for query without variables', () => { + const result = getQueryExprVariableRegExp('fieldName:value'); + expect(result).toBeNull(); + }); + + it('should return null for empty string', () => { + const result = getQueryExprVariableRegExp(''); + expect(result).toBeNull(); + }); + + it('should return null for string with only spaces', () => { + const result = getQueryExprVariableRegExp(' '); + expect(result).toBeNull(); + }); + + it('should find first variable when multiple variables exist', () => { + const result = getQueryExprVariableRegExp('field1:~$var1 field2:~$var2'); + expect(result?.[0]).toEqual('field1:~$var1'); + }); + + it('should find variable with special characters in field name', () => { + const result = getQueryExprVariableRegExp('field-name:~$var'); + expect(result?.[0]).toEqual('field-name:~$var'); + }); + + it('should not handle variable without field name prefix', () => { + const result = getQueryExprVariableRegExp('~$var'); + expect(result).toBeNull(); + }); + + it('should handle multiple spaces around variable', () => { + const result = getQueryExprVariableRegExp(' fieldName:~$var '); + expect(result?.[0]).toEqual(' fieldName:~$var'); + }); + + it('should find variable with mixed case field name', () => { + const result = getQueryExprVariableRegExp('fieldName:~$MyVariable'); + expect(result?.[0]).toEqual('fieldName:~$MyVariable'); + }); + + it('should find variable with dot separated field name', () => { + const result = getQueryExprVariableRegExp('someField: Some value | field.name.with.some.dot.separated:~$MyVariable'); + expect(result?.[0]).toEqual(' field.name.with.some.dot.separated:~$MyVariable'); + }); +}); diff --git a/src/LogsQL/regExpOperator.ts b/src/LogsQL/regExpOperator.ts new file mode 100644 index 00000000..21be62f7 --- /dev/null +++ b/src/LogsQL/regExpOperator.ts @@ -0,0 +1,8 @@ +/** + * Regular expression for matching variables in a query expression -> filterName:~"$VariableName" + * */ +const variableRegExp = /(^|\||\s)([^\s:~]+)\s*:\s*~\s*(?:"\$([A-Za-z0-9_.-]+)"|\$([A-Za-z0-9_.-]+)|"([^"]+)"|([^\s|]+))(?=\s|\||$)/; + +export function getQueryExprVariableRegExp(queryExpr: string) { + return variableRegExp.exec(queryExpr); +} diff --git a/src/components/QueryEditor/QueryEditor.tsx b/src/components/QueryEditor/QueryEditor.tsx index 2bccf1d3..eb5a145c 100644 --- a/src/components/QueryEditor/QueryEditor.tsx +++ b/src/components/QueryEditor/QueryEditor.tsx @@ -1,10 +1,11 @@ import { css } from "@emotion/css"; import { isEqual } from 'lodash'; -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { CoreApp, GrafanaTheme2, LoadingState } from '@grafana/data'; import { Button, ConfirmModal, useStyles2 } from '@grafana/ui'; +import { getQueryExprVariableRegExp } from "../../LogsQL/regExpOperator"; import { isExprHasStatsPipeFunctions } from "../../LogsQL/statsPipeFunctions"; import { storeKeys } from "../../store/constants"; import store from "../../store/store"; @@ -16,14 +17,16 @@ import { QueryBuilderContainer } from "./QueryBuilder/QueryBuilderContainer"; import { QueryEditorModeToggle } from "./QueryBuilder/QueryEditorModeToggle"; import { buildVisualQueryFromString } from "./QueryBuilder/utils/parseFromString"; import QueryCodeEditor from "./QueryCodeEditor"; +import { QueryEditorHelp } from "./QueryEditorHelp"; import { QueryEditorOptions } from "./QueryEditorOptions"; +import QueryEditorVariableRegexpError from "./QueryEditorVariableRegexpError"; import VmuiLink from "./VmuiLink"; import { changeEditorMode, getQueryWithDefaults } from "./state"; const QueryEditor = React.memo((props) => { const styles = useStyles2(getStyles); - const { onChange, onRunQuery, data, app, queries, datasource, range: timeRange } = props; + const { onChange, onRunQuery: runQuery, data, app, queries, datasource, range: timeRange } = props; const [dataIsStale, setDataIsStale] = useState(false); const [parseModalOpen, setParseModalOpen] = useState(false); @@ -31,6 +34,9 @@ const QueryEditor = React.memo((props) => { const editorMode = query.editorMode!; const isStatsQuery = query.queryType === QueryType.Stats || query.queryType === QueryType.StatsRange; const showStatsWarn = isStatsQuery && !isExprHasStatsPipeFunctions(query.expr || ''); + const varRegExp= useMemo(() => { + return getQueryExprVariableRegExp(query.expr)?.[0] || null; + }, [query.expr]); const onEditorModeChange = useCallback((newEditorMode: QueryEditorMode) => { if (newEditorMode === QueryEditorMode.Builder) { @@ -56,6 +62,13 @@ const QueryEditor = React.memo((props) => { onChange(query); }; + const onRunQuery = useCallback(() => { + if(varRegExp) { + return; + } + runQuery(); + }, [runQuery, varRegExp]); + useEffect(() => { // grafana with a version below 12 doesn't support subscribe function on store if ('subscribe' in store) { @@ -81,6 +94,7 @@ const QueryEditor = React.memo((props) => {
{showStatsWarn && ()} + ((props) => { ) : ( )} + {varRegExp && ()} { + const helpTooltipContent = useMemo(() => { + const helpText = `Variable Interpolation Rules: + +1. Field Value Variables: + • field:$var -> field:in("v1", ..., "vN") + • field:=$var -> field:in("v1", ..., "vN") + • Values are quoted and escaped + • Empty values or "All" expand to in(*) + +2. Function Contexts: + • in($var) and contains_any($var) expand to quoted lists + • Values maintain proper quoting within function calls + +3. Nequality Operators in stream filters: + • field:!$var -> !field in("v1", ..., "vN") + • field:!=$var -> !field in("v1", ..., "vN") + +4. Stream Filters: + • {tag=$var} -> {tag in(...)} + • {field!=$var} -> {tag not_in(...)} + +5. Restrictions: + • Interpolation is NOT allowed in regexp (e.g., field:~$var) + • Invalid usage will show an error message`; + return 
{helpText}
+ }, []); + + return ( + + + + ) +} diff --git a/src/components/QueryEditor/QueryEditorVariableRegexpError.tsx b/src/components/QueryEditor/QueryEditorVariableRegexpError.tsx new file mode 100644 index 00000000..d1d707dc --- /dev/null +++ b/src/components/QueryEditor/QueryEditorVariableRegexpError.tsx @@ -0,0 +1,44 @@ +import { css } from "@emotion/css"; +import React from 'react'; + +import { GrafanaTheme2 } from "@grafana/data"; +import { Badge, useTheme2 } from "@grafana/ui"; + + +interface Props { + regExp: string; +} + +const QueryEditorVariableRegexpError = ({ regExp }: Props) => { + const theme = useTheme2(); + const styles = getStyles(theme); + + const text = ( +
+ Regexp operator (~) cannot be used with variables in {regExp}. Use exact match operator (:) or use non variable in regexp instead. +
+ ) + + return ( +
+ +
+ ) +} + +export default QueryEditorVariableRegexpError; + +const getStyles = (theme: GrafanaTheme2) => { + return { + root: css({ + display: 'flex', + alignItems: 'center', + gap: theme.spacing(1), + flexGrow: 1, + }), + }; +}; diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 00000000..dd2fe9ee --- /dev/null +++ b/src/constants.ts @@ -0,0 +1 @@ +export const VARIABLE_ALL_VALUE = "$__all"; diff --git a/src/datasource.test.ts b/src/datasource.test.ts index 6a0268b2..5ea63e94 100644 --- a/src/datasource.test.ts +++ b/src/datasource.test.ts @@ -1,13 +1,15 @@ -import { AdHocVariableFilter } from '@grafana/data'; +import { AdHocVariableFilter } from '@grafana/data'; import { TemplateSrv } from "@grafana/runtime"; import { createDatasource } from "./__mocks__/datasource"; +import { VARIABLE_ALL_VALUE } from "./constants"; import { VictoriaLogsDatasource } from "./datasource"; - + const replaceMock = jest.fn().mockImplementation((a: string) => a); const templateSrvStub = { replace: replaceMock, + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; beforeEach(() => { @@ -45,6 +47,10 @@ describe('VictoriaLogsDatasource', () => { it('should return a number for numeric value', () => { expect(ds.interpolateQueryExpr(1000 as any, customVariable)).toEqual(1000); }); + + it('should return a value escaped by stringify for one array element', () => { + expect(ds.interpolateQueryExpr(['arg // for & test " this string ` end test'] as any, customVariable)).toEqual("arg // for & test \" this string ` end test"); + }); }); describe('applyTemplateVariables', () => { @@ -90,6 +96,7 @@ describe('VictoriaLogsDatasource', () => { const scopedVars = {}; const templateSrvMock = { replace: jest.fn((a: string) => a), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -105,6 +112,7 @@ describe('VictoriaLogsDatasource', () => { }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', '"bar"')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -121,6 +129,7 @@ describe('VictoriaLogsDatasource', () => { const replaceValue = `$_StartMultiVariable_${scopedVars.var.value.join("_separator_")}_EndMultiVariable` const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', replaceValue)), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -136,6 +145,7 @@ describe('VictoriaLogsDatasource', () => { }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', '("foo" OR "bar")')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -151,6 +161,7 @@ describe('VictoriaLogsDatasource', () => { }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', '"0.0.0.0:3000"')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -162,10 +173,14 @@ describe('VictoriaLogsDatasource', () => { it('should correctly substitute an array of URLs into an OR expression', () => { const scopedVars = { - var: { text: 'http://localhost:3001/,http://192.168.50.60:3000/foo', value: ['http://localhost:3001/', 'http://192.168.50.60:3000/foo'] }, + var: { + text: 'http://localhost:3001/,http://192.168.50.60:3000/foo', + value: ['http://localhost:3001/', 'http://192.168.50.60:3000/foo'] + }, }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', '("http://localhost:3001/" OR "http://192.168.50.60:3000/foo")')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -181,6 +196,7 @@ describe('VictoriaLogsDatasource', () => { }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var', '')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -197,6 +213,7 @@ describe('VictoriaLogsDatasource', () => { }; const templateSrvMock = { replace: jest.fn((a: string) => a?.replace('$var1', '"foo"').replace('$var2', '"bar"')), + getVariables: jest.fn().mockReturnValue([]), } as unknown as TemplateSrv; const ds = createDatasource(templateSrvMock); const replacedQuery = ds.applyTemplateVariables( @@ -222,4 +239,32 @@ describe('VictoriaLogsDatasource', () => { expect(result).toBe('key1:="value1" AND key2:!="value2"'); }); }); + + describe('interpolateString', () => { + it('should interpolate string with all and multi values', () => { + const scopedVars = {}; + const variables = [ + { + name: 'var1', + current: [{ value: "foo" }, { value: "bar" }], + multi: true, + type: "query", + query: { + type: "fieldValue" + } + }, { + name: 'var2', + current: { value: VARIABLE_ALL_VALUE }, + multi: false, + } + ]; + const templateSrvMock = { + replace: jest.fn(() => 'foo: in($_StartMultiVariable_foo_separator_bar_EndMultiVariable) bar: in(*)'), + getVariables: jest.fn().mockReturnValue(variables), + } as unknown as TemplateSrv; + const ds = createDatasource(templateSrvMock); + const result = ds.interpolateString('foo: $var1 bar: $var2', scopedVars); + expect(result).toStrictEqual('foo: in(\"foo\",\"bar\") bar: in(*)'); + }) + }); }); diff --git a/src/datasource.ts b/src/datasource.ts index a34c45a9..22bdbfa0 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -20,12 +20,14 @@ import { LogRowContextQueryDirection, LogRowModel, MetricFindValue, + QueryVariableModel, rangeUtil, ScopedVars, SupplementaryQueryOptions, SupplementaryQueryType, TimeRange, toUtc, + TypedVariableModel, } from '@grafana/data'; import { config, DataSourceWithBackend, getGrafanaLiveSrv, getTemplateSrv, TemplateSrv, } from '@grafana/runtime'; import { DataQuery } from "@grafana/schema"; @@ -33,11 +35,12 @@ import { DataQuery } from "@grafana/schema"; import { transformBackendResult } from "./backendResultTransformer"; import QueryEditor from "./components/QueryEditor/QueryEditor"; import { LogLevelRule } from "./configuration/LogLevelRules/types"; +import { VARIABLE_ALL_VALUE } from "./constants"; import { escapeLabelValueInSelector } from "./languageUtils"; import LogsQlLanguageProvider from "./language_provider"; import { LOGS_VOLUME_BARS, queryLogsVolume } from "./logsVolumeLegacy"; import { addLabelToQuery, addSortPipeToQuery, queryHasFilter, removeLabelFromQuery } from "./modifyQuery"; -import { returnVariables } from "./parsingUtils"; +import { replaceOperatorWithIn, returnVariables } from "./parsingUtils"; import { storeKeys } from "./store/constants"; import store from "./store/store"; import { @@ -206,10 +209,10 @@ export class VictoriaLogsDatasource interpolateQueryExpr(value: any, _variable: any) { if (typeof value === 'string') { - return value; + return value ? JSON.stringify(value) : value; } - if (Array.isArray(value) && value.length > 1) { + if (Array.isArray(value)) { return value.length > 1 ? `$_StartMultiVariable_${value.join("_separator_")}_EndMultiVariable` : value[0] || ""; } @@ -270,15 +273,37 @@ export class VictoriaLogsDatasource : [] } + isAllOption(variable: TypedVariableModel): boolean { + const value = 'current' in variable && variable?.current?.value; + if (!value) { + return false; + } + return Array.isArray(value) ? value.includes(VARIABLE_ALL_VALUE) : value === VARIABLE_ALL_VALUE; + } + + replaceOperatorsToInForMultiQueryVariables(expr: string,) { + const variables = this.templateSrv.getVariables(); + const fieldValuesVariables = variables.filter(v => v.type === 'query' && v.query.type === 'fieldValue' && v.multi || this.isAllOption(v)) as QueryVariableModel[]; + let result = expr; + for (let variable of fieldValuesVariables) { + result = replaceOperatorWithIn(result, variable.name); + if (this.isAllOption(variable)) { + result = result.replace(`$${variable.name}`, '*'); + } + } + return result; + } + interpolateString(string: string, scopedVars?: ScopedVars) { - const expr = this.templateSrv.replace(string, scopedVars, this.interpolateQueryExpr); + const exprWithReplacedOperators = this.replaceOperatorsToInForMultiQueryVariables(string); + const expr = this.templateSrv.replace(exprWithReplacedOperators, scopedVars, this.interpolateQueryExpr); return this.replaceMultiVariables(expr) } private replaceMultiVariables(input: string): string { const multiVariablePattern = /["']?\$_StartMultiVariable_(.+?)_EndMultiVariable["']?/g; - return input.replace(multiVariablePattern, (match, valueList, offset) => { + return input.replace(multiVariablePattern, (match, valueList: string, offset) => { const values = valueList.split('_separator_'); const precedingChars = input.slice(0, offset).replace(/\s+/g, '').slice(-3); @@ -286,7 +311,7 @@ export class VictoriaLogsDatasource if (precedingChars.includes("~")) { return `"(${values.join("|")})"`; } else if (precedingChars.includes("in(")) { - return values.join(","); + return values.map(value => JSON.stringify(value)).join(","); } return values.join(" OR "); }); diff --git a/src/parsingUtils.test.ts b/src/parsingUtils.test.ts new file mode 100644 index 00000000..ff3999d3 --- /dev/null +++ b/src/parsingUtils.test.ts @@ -0,0 +1,360 @@ +// parsingUtils.test.ts +import { replaceOperatorWithIn } from './parsingUtils'; + +describe('replaceOperatorWithIn', () => { + describe('one var', () => { + const operators = [':', ':=']; + operators.forEach(operator => { + describe(`${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('field1:in($variableName)'); + }); + + it(`should replace ${operator} operator right space with ":in()" syntax`, () => { + const input = `field1${operator} $variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('field1:in($variableName)'); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('field1:in($variableName)'); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1${operator}$variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('field1:in($variableName)'); + }); + }); + }) + + const negativeOperators = [':!', ':!=']; + negativeOperators.forEach(operator => { + describe(`negative ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('!field1:in($variableName)'); + }); + + it(`should replace ${operator} operator right space with ":in()" syntax`, () => { + const input = `field1${operator} $variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('!field1:in($variableName)'); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('!field1:in($variableName)'); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1${operator}$variableName`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('!field1:in($variableName)'); + }); + }); + }); + + const streamOperators = ['=']; + streamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 in($variableName)}'); + }); + + it(`should replace ${operator} operator right space with "in()" syntax`, () => { + const input = `{field1${operator} $variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 in($variableName)}'); + }); + + it(`should replace ${operator} operator with left space with "in()" syntax`, () => { + const input = `{field1 ${operator}$variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 in($variableName)}'); + }); + + it(`should replace ${operator} operator without spaces with "in()" syntax`, () => { + const input = `{field1${operator}$variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 in($variableName)}'); + }); + }); + }); + + const negativeStreamOperators = ['!=']; + negativeStreamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 not_in($variableName)}'); + }); + + it(`should replace ${operator} operator right space with "in()" syntax`, () => { + const input = `{field1${operator} $variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 not_in($variableName)}'); + }); + + it(`should replace ${operator} operator with left space with "in()" syntax`, () => { + const input = `{field1 ${operator}$variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 not_in($variableName)}'); + }); + + it(`should replace ${operator} operator without spaces with "in()" syntax`, () => { + const input = `{field1${operator}$variableName}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe('{field1 not_in($variableName)}'); + }); + }); + }); + }); + + describe('two vars with first var replacement', () => { + const variableName = 'variableName1'; + const operators = [':', ':=']; + operators.forEach(operator => { + describe(`${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator right space with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1${operator}$variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1:in($variableName1) field2 ${operator} $variableName2`); + }); + }); + }); + + const negativeOperators = [':!', ':!=']; + negativeOperators.forEach(operator => { + describe(`${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`!field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator with right space with ":in()" syntax`, () => { + const input = `field1${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`!field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`!field1:in($variableName1) field2 ${operator} $variableName2`); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1${operator}$variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`!field1:in($variableName1) field2 ${operator} $variableName2`); + }); + }); + }); + + const streamOperators = ['=']; + streamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator right space with "in()" syntax`, () => { + const input = `{field1${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator with left space with "in()" syntax`, () => { + const input = `{field1 ${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator without spaces with "in()" syntax`, () => { + const input = `{field1${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 in($variableName) field2 ${operator} $variableName2}`); + }); + }); + }); + + const negativeStreamOperators = ['!=']; + negativeStreamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 not_in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator right space with "in()" syntax`, () => { + const input = `{field1${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 not_in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator with left space with "in()" syntax`, () => { + const input = `{field1 ${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 not_in($variableName) field2 ${operator} $variableName2}`); + }); + + it(`should replace ${operator} operator without spaces with "in()" syntax`, () => { + const input = `{field1${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, 'variableName'); + expect(output).toBe(`{field1 not_in($variableName) field2 ${operator} $variableName2}`); + }); + }); + }); + }); + + describe('two vars with second var replacement', () => { + const variableName = 'variableName2'; + const operators = [':', ':=']; + operators.forEach(operator => { + describe(`${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator} $variableName1 field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator right space with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator} $variableName1 field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName1 field2 ${operator}$variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator}$variableName1 field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1${operator}$variableName1 field2${operator}$variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1${operator}$variableName1 field2:in($variableName2)`); + }); + }); + }) + + const negativeOperators = [':!', ':!=']; + negativeOperators.forEach(operator => { + describe(`${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2 ${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator} $variableName1 !field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator with right space with ":in()" syntax`, () => { + const input = `field1${operator}$variableName1 field2${operator} $variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1${operator}$variableName1 !field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator with left space with ":in()" syntax`, () => { + const input = `field1 ${operator}$variableName1 field2 ${operator}$variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator}$variableName1 !field2:in($variableName2)`); + }); + + it(`should replace ${operator} operator without spaces with ":in()" syntax`, () => { + const input = `field1 ${operator} $variableName1 field2${operator}$variableName2`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`field1 ${operator} $variableName1 !field2:in($variableName2)`); + }); + }); + }); + + const streamOperators = ['=']; + streamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1 ${operator} $variableName field2 in($variableName2)}`); + }); + + it(`should replace ${operator} operator right space with "in()" syntax`, () => { + const input = `{field1${operator} $variableName field2${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1${operator} $variableName field2 in($variableName2)}`); + }); + + it(`should replace ${operator} operator with left space with "in()" syntax`, () => { + const input = `{field1 ${operator} $variableName field2 ${operator}$variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1 ${operator} $variableName field2 in($variableName2)}`); + }); + + it(`should replace ${operator} operator without spaces with "in()" syntax`, () => { + const input = `{field1${operator}$variableName field2${operator}$variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1${operator}$variableName field2 in($variableName2)}`); + }); + }); + }); + + const negativeStreamOperators = ['!=']; + negativeStreamOperators.forEach(operator => { + describe(`stream ${operator} cases`, () => { + it(`should replace ${operator} operator with spaces with "not_in()" syntax`, () => { + const input = `{field1 ${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1 ${operator} $variableName field2 not_in($variableName2)}`); + }); + + it(`should replace ${operator} operator right space with "not_in()" syntax`, () => { + const input = `{field1${operator} $variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1${operator} $variableName field2 not_in($variableName2)}`); + }); + + it(`should replace ${operator} operator with left space with "not_in()" syntax`, () => { + const input = `{field1 ${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1 ${operator}$variableName field2 not_in($variableName2)}`); + }); + + it(`should replace ${operator} operator without spaces with "not_in()" syntax`, () => { + const input = `{field1${operator}$variableName field2 ${operator} $variableName2}`; + const output = replaceOperatorWithIn(input, variableName); + expect(output).toBe(`{field1${operator}$variableName field2 not_in($variableName2)}`); + }); + }); + }); + }); +}); diff --git a/src/parsingUtils.ts b/src/parsingUtils.ts index 4ab471d7..555e411a 100644 --- a/src/parsingUtils.ts +++ b/src/parsingUtils.ts @@ -31,3 +31,248 @@ export function replaceVariables(expr: string) { return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : ''); }); } + +const validAfterVariableChars = [' ', '|', '}']; +function findIndexEndOfFilter(expr: string, startIndex = 0): number { + for (let i = startIndex; i < expr.length; i++) { + if (validAfterVariableChars.includes(expr[i])) { + return i; + } + } + return -1; +} + +enum OperatorType { + NEGATED_EQUALS = 'NEGATED_EQUALS', // :! or :!= or != + EQUALS = 'EQUALS', // : or := or = + NONE = 'NONE' +} + +interface OperatorInfo { + startOperatorIndex: number; + type: OperatorType; + length: number; + leftSpaces: number; + rightSpaces: number; + isStreamOperator?: boolean; +} + +function isValidVariablePosition(char: string | undefined): boolean { + return !char || validAfterVariableChars.includes(char); +} + +function shouldSkipVariable(result: string, varIndex: number, varPattern: string): boolean { + if (varIndex === 0) { + return true; + } + + const charAfterVar = result[varIndex + varPattern.length]; + return !isValidVariablePosition(charAfterVar); +} + +function countLeftSpaces(str: string, endPos: number): number { + let count = 0; + for (let i = endPos - 1; i >= 0 && str[i] === ' '; i--) { + count++; + } + return count; +} + +function detectStreamOperator(queryExpr: string, endOperatorPos: number, rightSpaces: number): null | OperatorInfo { + if (queryExpr[endOperatorPos - 1] !== '=') { + return null; + } + + if (queryExpr[endOperatorPos - 2] === '!' && queryExpr[endOperatorPos - 3] !== ':') { + const startOperatorIndex = endOperatorPos - 2; + const leftSpaces = countLeftSpaces(queryExpr, startOperatorIndex); + return { + type: OperatorType.NEGATED_EQUALS, + length: 2, + leftSpaces, + rightSpaces, + startOperatorIndex, + isStreamOperator: true + }; + } + + const invalidPreChars = [':', '!']; + if (!invalidPreChars.includes(queryExpr[endOperatorPos - 2])) { + const startOperatorIndex = endOperatorPos - 1; + const leftSpaces = countLeftSpaces(queryExpr, startOperatorIndex); + return { + type: OperatorType.EQUALS, + length: 1, + leftSpaces, + rightSpaces, + startOperatorIndex, + isStreamOperator: true + }; + } + + return null; +} + +/** + * Detects the operator in the given query expression near a specified variable index. + * + * @param {string} queryExpr - The query expression string to analyze. + * @param {number} varIndex - The index of the variable or reference point in the query expression to check for an operator. + * @return {OperatorInfo} An object representing details about the detected operator, including its type, length, spaces around it, and its starting index. + */ +function detectOperator(queryExpr: string, varIndex: number): OperatorInfo { + const rightSpaces = countLeftSpaces(queryExpr, varIndex); + let endOperatorPos = varIndex - rightSpaces; + + // for operator "!=" | "=" + const streamOperator = detectStreamOperator(queryExpr, endOperatorPos, rightSpaces); + if (streamOperator) { + return streamOperator; + } + + // for operator ":" + let startOperatorIndex = endOperatorPos - 1; + let length = 1; + let leftSpaces = countLeftSpaces(queryExpr, startOperatorIndex); + const oneCharBeforeSpaces = queryExpr[endOperatorPos - 1]; + if (oneCharBeforeSpaces === ':' ) { + return { type: OperatorType.EQUALS, length, leftSpaces, rightSpaces, startOperatorIndex }; + } + + // for operator ":=" | ":!" + startOperatorIndex = endOperatorPos - 2; + length = 2; + leftSpaces = countLeftSpaces(queryExpr, startOperatorIndex); + const twoCharsBeforeSpaces = queryExpr.slice(Math.max(0, startOperatorIndex), endOperatorPos); + const twoCharsOperators = [':=', ':!'] + if (twoCharsOperators.includes(twoCharsBeforeSpaces)) { + const type = twoCharsBeforeSpaces.includes('!') ? OperatorType.NEGATED_EQUALS : OperatorType.EQUALS; + return { type, length, leftSpaces, rightSpaces, startOperatorIndex }; + } + + // for operator ":!=" + startOperatorIndex = endOperatorPos - 3; + length = 3; + leftSpaces = countLeftSpaces(queryExpr, startOperatorIndex); + const threeCharsBeforeSpaces = queryExpr.slice(Math.max(0, startOperatorIndex), endOperatorPos); + const threeCharsOperators = [':!='] + if (threeCharsOperators.includes(threeCharsBeforeSpaces)) { + return { type: OperatorType.NEGATED_EQUALS, length, leftSpaces, rightSpaces, startOperatorIndex }; + } + + return { type: OperatorType.NONE, length: 0, leftSpaces: 0, rightSpaces: 0, startOperatorIndex: -1 }; +} + +function findFieldName(result: string, operatorStart: number): { fieldName: string; fieldStart: number } { + let fieldStart = operatorStart - 1; + while (fieldStart >= 0 && result[fieldStart] !== ' ' && result[fieldStart] !== '|') { + fieldStart--; + } + fieldStart++; + + const fieldName = result.slice(fieldStart, operatorStart); + return { fieldName, fieldStart }; +} + +function transformNegatedOperator( + result: string, + varIndex: number, + varPattern: string, + operatorInfo: OperatorInfo, + actualEndIndex: number +): { transformed: string; newSearchStart: number } { + const totalSpaces = operatorInfo.leftSpaces + operatorInfo.rightSpaces; + const operatorStart = varIndex - operatorInfo.length - totalSpaces; + const afterVariable = result.slice(actualEndIndex); + + const { fieldName, fieldStart } = findFieldName(result, operatorStart); + const beforeField = result.slice(0, fieldStart); + + let filterPart: string; + if(operatorInfo.isStreamOperator){ + filterPart = `${fieldName.trimEnd()} not_in(${varPattern})` + } else { + filterPart = `!${fieldName.trimEnd()}:in(${varPattern})` + } + + const transformed = `${beforeField}${filterPart}${afterVariable}`; + const newSearchStart = beforeField.length + filterPart.length; + + return { transformed, newSearchStart }; +} + +function transformWithOperator( + result: string, + varIndex: number, + varPattern: string, + operatorInfo: OperatorInfo, + actualEndIndex: number +): { transformed: string; newSearchStart: number } { + const totalSpaces = operatorInfo.leftSpaces + operatorInfo.rightSpaces; + const operatorStart = varIndex - operatorInfo.length - totalSpaces; + const beforeOperator = result.slice(0, operatorStart); + const afterVariable = result.slice(actualEndIndex); + const inOperator = operatorInfo.isStreamOperator ? ' in' : ':in'; + + const filterWithBeforeOperatorPart = `${beforeOperator.trimEnd()}${inOperator}(${varPattern})`; + const transformed = filterWithBeforeOperatorPart + afterVariable; + + return { transformed, newSearchStart: filterWithBeforeOperatorPart.length }; +} + +/** + * Replaces occurrences of a specific variable `variableName` within the operator 'in' in a given expression. + * + * @param {string} expr - The expression in which variable patterns and operators need to be replaced or transformed. + * @param {string} variableName - The name of the variable to look for in the expression, prefixed with a `$` symbol. + * @return {string} - The transformed expression with the specified variable patterns replaced according to the defined rules. + */ +export function replaceOperatorWithIn(expr: string, variableName: string): string { + const varPattern = `$${variableName}`; + let result = expr; + let searchStart = 0; + + while (true) { + const varIndex = result.indexOf(varPattern, searchStart); + + if (varIndex === -1) { + break; + } + + if (shouldSkipVariable(result, varIndex, varPattern)) { + searchStart = varIndex + varPattern.length; + continue; + } + + const filterEndIndex = findIndexEndOfFilter(result, varIndex + varPattern.length); + const actualEndIndex = filterEndIndex === -1 ? result.length : filterEndIndex; + + const operatorInfo = detectOperator(result, varIndex); + + if (operatorInfo.type === OperatorType.NEGATED_EQUALS) { + const { transformed, newSearchStart } = transformNegatedOperator( + result, + varIndex, + varPattern, + operatorInfo, + actualEndIndex + ); + result = transformed; + searchStart = newSearchStart; + } else if (operatorInfo.type === OperatorType.EQUALS) { + const { transformed, newSearchStart } = transformWithOperator( + result, + varIndex, + varPattern, + operatorInfo, + actualEndIndex + ); + result = transformed; + searchStart = newSearchStart; + } else { + searchStart = varIndex + varPattern.length; + } + } + + return result; +}