Add customizable adhoc SQL query for select adhoc values

.eslintrc

@@ -1,9 +1,6 @@
 {
   "extends": "./.config/.eslintrc",
   "rules": {
-    "array-element-newline": ["error", {
-      "ArrayPattern": { "minItems": 3 }
-    }],
     "object-curly-newline": ["error", {
       "ObjectPattern": { "multiline": true, "minProperties": 10 }
     }]

src/datasource/adhoc.ts

@@ -1,8 +1,10 @@
+export const DEFAULT_VALUES_QUERY = 'SELECT DISTINCT {field} AS value FROM {database}.{table} LIMIT 300';
 export default class AdHocFilter {
   tagKeys: any[];
   tagValues: { [key: string]: any } = {};
   datasource: any;
   query: string;
+  adHocValuesQuery: string;
 
   constructor(datasource: any) {
     const queryFilter = "database NOT IN ('system','INFORMATION_SCHEMA','information_schema')";
@@ -12,6 +14,7 @@ export default class AdHocFilter {
     this.tagKeys = [];
     this.tagValues = [];
     this.datasource = datasource;
+    this.adHocValuesQuery = datasource.adHocValuesQuery
     let filter = queryFilter;
     if (datasource.defaultDatabase.length > 0) {
       filter = "database = '" + datasource.defaultDatabase + "' AND " + queryFilter;
@@ -43,9 +46,8 @@ export default class AdHocFilter {
     response.forEach((item: any) => {
       const databasePrefix = this.datasource.defaultDatabase.length === 0 ? item.database + '.' : '';
       const text: string = databasePrefix + item.table + '.' + item.name;
-      const value = item.name;
 
-      this.tagKeys.push({ text, value });
+      this.tagKeys.push({ text: text, value: text });
 
       if (item.type.slice(0, 4) === 'Enum') {
         const regexEnum = /'(?:[^']+|'')+'/gim;
@@ -71,39 +73,55 @@ export default class AdHocFilter {
   // GetTagValues returns column values according to passed options
   // Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues`
   // Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues`
-  GetTagValues(options: any) {
-    const valuesQuery = 'SELECT DISTINCT {field} AS value FROM {database}.{table} LIMIT 300';
+  GetTagValues(options) {
+    // Determine which query to use initially
+    const initialQuery = this.adHocValuesQuery || DEFAULT_VALUES_QUERY;
 
-    let self = this;
-    if (this.tagValues.hasOwnProperty(options.key)) {
+    // If the tag values are already cached, return them immediately
+    if (Object.prototype.hasOwnProperty.call(this.tagValues, options.key)) {
       return Promise.resolve(this.tagValues[options.key]);
     }
-    let key_items = options.key.split('.');
+    // Split the key to extract database, table, and field
+    const keyItems = options.key.split('.');
     if (
-      key_items.length < 2 ||
-      (key_items.length === 2 && this.datasource.defaultDatabase.length === 0) ||
-      key_items.length > 3
+      keyItems.length < 2 ||
+      (keyItems.length === 2 && !this.datasource.defaultDatabase) ||
+      keyItems.length > 3
     ) {
       return Promise.resolve([]);
     }
-    let field, database, table;
-    if (key_items.length === 3) {
+
+    // Destructure key items based on their length
+    let database, table, field;
+    if (keyItems.length === 3) {
       [
         database,
         table,
-        field
-      ] = key_items;
-    }
-    if (key_items.length === 2) {
-      database = self.datasource.defaultDatabase;
-      [table, field] = key_items;
+        field] = keyItems;
+    } else {
+      database = this.datasource.defaultDatabase;
+      [table, field] = keyItems;
     }
-    let q = valuesQuery.replace('{field}', field).replace('{database}', database).replace('{table}', table);
 
-    return this.datasource.metricFindQuery(q).then(function (response: any) {
-      self.tagValues[options.key] = self.processTagValuesResponse(response);
-      return self.tagValues[options.key];
-    });
+    // Function to build the query
+    const buildQuery = (queryTemplate) =>
+      queryTemplate
+        .replace('{field}', field)
+        .replace('{database}', database)
+        .replace('{table}', table);
+
+    // Execute the initial query
+    return this.datasource.metricFindQuery(buildQuery(initialQuery))
+      .then((response: any) => {
+        // Process and cache the response
+        this.tagValues[options.key] = this.processTagValuesResponse(response);
+        return this.tagValues[options.key];
+      })
+      .catch((error: any) => {
+        this.tagValues[options.key] = [];
+        console.error(error);
+        return this.tagValues[options.key];
+      });
   }
 
   processTagValuesResponse(response: any) {

src/datasource/datasource.ts

@@ -22,6 +22,7 @@ import { CHDataSourceOptions, CHQuery, DEFAULT_QUERY } from '../types/types';
 import { SqlQueryHelper } from './sql-query/sql-query-helper';
 import SqlQueryMacros from './sql-query/sql-query-macros';
 import { QueryEditor } from "../views/QueryEditor/QueryEditor";
+import {getAdhocFilters} from "../views/QueryEditor/helpers/getAdHocFilters";
 
 const adhocFilterVariable = 'adhoc_query_filter';
 export
@@ -46,15 +47,19 @@ export
   useYandexCloudAuthorization: boolean;
   useCompression: boolean;
   compressionType: string;
+  adHocValuesQuery: string;
+  uid: string;
 
   constructor(instanceSettings: DataSourceInstanceSettings<CHDataSourceOptions>) {
     super(instanceSettings);
+    this.uid = instanceSettings.uid;
     this.url = instanceSettings.url!;
     this.basicAuth = instanceSettings.basicAuth;
     this.withCredentials = instanceSettings.withCredentials;
     this.addCorsHeader = instanceSettings.jsonData.addCorsHeader || false;
     this.usePOST = instanceSettings.jsonData.usePOST || false;
     this.useCompression = instanceSettings.jsonData.useCompression || false;
+    this.adHocValuesQuery = instanceSettings.jsonData.adHocValuesQuery || '';
     this.compressionType = instanceSettings.jsonData.compressionType || '';
     this.defaultDatabase = instanceSettings.jsonData.defaultDatabase || '';
     this.xHeaderUser = instanceSettings.jsonData.xHeaderUser || '';
@@ -411,7 +416,7 @@ export
   createQuery(options: any, target: any) {
     const queryModel = new SqlQuery(target, this.templateSrv, options);
     // @ts-ignore
-    const adhocFilters = this.templateSrv?.getAdhocFilters(this.adHocFilter?.datasource?.name || 'clickhouse') || []
+    const adhocFilters = getAdhocFilters(this.adHocFilter?.datasource?.name || 'clickhouse',this.uid )
     const stmt = queryModel.replace(options, adhocFilters);
 
     let keys = [];

src/datasource/sql-query/sql_query.ts

@@ -43,7 +43,6 @@ export default class SqlQuery {
     let intervalMs = SqlQueryHelper.convertInterval(i, this.target.intervalFactor || 1, true);
     let adhocCondition: any[] = [];
 
-    adhocFilters = this.target.adHocFilters
     try {
       let ast = scanner.toAST();
       let topQueryAST = ast;

src/spec/datasource.jest.ts

@@ -183,13 +183,13 @@ describe('clickhouse sql series:', () => {
       let results = adhocCtrl.tagKeys;
       expect(results.length).toBe(6);
       expect(results[0].text).toBe('requests.Event');
-      expect(results[0].value).toBe('Event');
+      expect(results[0].value).toBe('requests.Event');
 
       expect(results[1].text).toBe('requests.UserID');
-      expect(results[1].value).toBe('UserID');
+      expect(results[1].value).toBe('requests.UserID');
 
       expect(results[2].text).toBe('requests.URL');
-      expect(results[2].value).toBe('URL');
+      expect(results[2].value).toBe('requests.URL');
 
       expect(results[3].text).toBe('Event');
       expect(results[3].value).toBe('Event');

src/types/types.ts

@@ -43,6 +43,7 @@ export interface CHQuery extends DataQuery {
   interval?: string;
   formattedQuery?: string;
   contextWindowSize?: number;
+  adHocValuesQuery?: string;
 }
 
 /**
@@ -64,6 +65,7 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
   defaultUint32?: string;
   defaultDateDate32?: string;
   defaultDateTimeType?: string;
+  adHocValuesQuery: string;
 }
 
 /**

src/views/ConfigEditor/ConfigEditor.tsx

@@ -1,9 +1,13 @@
 import React, { FormEvent, useState } from 'react';
-import { DataSourceHttpSettings, InlineField, InlineSwitch, Input, SecretInput, Select } from '@grafana/ui';
+import { CodeEditor, DataSourceHttpSettings, InlineField, InlineSwitch, Input, SecretInput, Select } from '@grafana/ui';
 import { DataSourcePluginOptionsEditorProps, onUpdateDatasourceJsonDataOption, SelectableValue } from '@grafana/data';
 import { CHDataSourceOptions } from '../../types/types';
 import _ from 'lodash';
 import { DefaultValues } from './FormParts/DefaultValues/DefaultValues';
+import { LANGUAGE_ID } from '../QueryEditor/components/QueryTextEditor/editor/initiateEditor';
+import { MONACO_EDITOR_OPTIONS } from '../constants';
+import {COMPRESSION_TYPE_OPTIONS} from "./constants";
+import {DEFAULT_VALUES_QUERY} from "../../datasource/adhoc";
 
 export interface CHSecureJsonData {
   password?: string;
@@ -128,7 +132,6 @@ export function ConfigEditor(props: Props) {
         newOptions={newOptions}
         onSwitchToggle={onSwitchToggle}
         onFieldChange={onFieldChange}
-        externalProps={props}
       />
       <h3 className="page-heading">Additional</h3>
       <div className="gf-form-group">
@@ -191,14 +194,26 @@ export function ConfigEditor(props: Props) {
             width={24}
             value={selectedCompressionType}
             onChange={({ value }) => onCompressionTypeChange({ value })}
-            options={[
-              { label: 'gzip', value: 'gzip' },
-              { label: 'br', value: 'br' },
-              { label: 'deflate', value: 'deflate' },
-              { label: 'zstd', value: 'zstd' },
-            ]}
+            options={COMPRESSION_TYPE_OPTIONS}
           />
         </InlineField>
+        <InlineField
+          label="Configure AdHoc Filters request"
+          labelWidth={32}
+          tooltip="To be able to configure request properfly please use macroses {field} {database} {table}"
+        >
+          <div style={{ position: 'relative', minWidth: '600px' }}>
+            <CodeEditor
+              height={Math.max((jsonData.adHocValuesQuery || '').split('\n').length * 18, 150)}
+              value={jsonData.adHocValuesQuery || DEFAULT_VALUES_QUERY}
+              language={LANGUAGE_ID}
+              monacoOptions={MONACO_EDITOR_OPTIONS}
+              onChange={(newValue) => {
+                onFieldChange({ value: newValue }, 'adHocValuesQuery');
+              }}
+            />
+          </div>
+        </InlineField>
       </div>
     </>
   );

src/views/ConfigEditor/FormParts/DefaultValues/DefaultValues.tsx

@@ -6,19 +6,19 @@ import { TimestampFormat } from '../../../../types/types';
 const TIME_RELATED_COLUMNS_QUERY =
   "SELECT name,database,table,type FROM system.columns WHERE substring(type,1,4) = 'Date' OR substring(type,10,4) = 'Date' OR type = 'UInt32' ORDER BY type,name FORMAT JSON";
 
+interface DefaultValuesInterface {
+  jsonData: any;
+  onSwitchToggle: any;
+  newOptions: any;
+  onFieldChange: any;
+}
+
 export const DefaultValues = ({
   jsonData,
   newOptions,
   onSwitchToggle,
   onFieldChange,
-  externalProps,
-}: {
-  jsonData: any;
-  onSwitchToggle: any;
-  newOptions: any;
-  onFieldChange: any;
-  externalProps: any;
-}) => {
+}: DefaultValuesInterface) => {
   const [defaultDateTime64Options, setDefaultDateTime64Options] = useState<any[]>([]);
   const [defaultDateTimeOptions, setDefaultDateTimeOptions] = useState<any[]>([]);
   const [defaultUint32Options, setDefaultUint32Options] = useState<any[]>([]);

src/views/ConfigEditor/constants.ts

@@ -0,0 +1,6 @@
+export const COMPRESSION_TYPE_OPTIONS = [
+  { label: 'gzip', value: 'gzip' },
+  { label: 'br', value: 'br' },
+  { label: 'deflate', value: 'deflate' },
+  { label: 'zstd', value: 'zstd' },
+]

src/views/QueryEditor/QueryEditor.tsx

@@ -10,10 +10,10 @@ import { useQueryState } from './hooks/useQueryState';
 import { useFormattedData } from './hooks/useFormattedData';
 import { initializeQueryDefaults } from './helpers/initializeQueryDefaults';
 import './QueryEditor.css';
+import {getAdhocFilters} from "./helpers/getAdHocFilters";
 
 export function QueryEditor(props: QueryEditorProps<CHDataSource, CHQuery, CHDataSourceOptions>) {
   const { datasource, query, onChange, onRunQuery } = props;
-
   const isAnnotationView = !props.app;
   const initializedQuery = initializeQueryDefaults(query, isAnnotationView, datasource, onChange);
   const [formattedData, error] = useFormattedData(initializedQuery, datasource);
@@ -25,19 +25,15 @@ export function QueryEditor(props: QueryEditorProps<CHDataSource, CHQuery, CHDat
   const onTriggerQuery = () => onRunQuery();
 
   // @ts-ignore
-  const adHocFilters = datasource.templateSrv.getAdhocFilters(datasource.name);
+  const adHocFilters = getAdhocFilters(datasource.name, query.datasource.uid)
   const areAdHocFiltersAvailable = !!adHocFilters.length;
+  useEffect(() => {
+    if (props.app !== 'explore') {
+      onChange({ ...initializedQuery, adHocFilters: adHocFilters });
+    }
 
-  if (adHocFilters?.length) {
     // eslint-disable-next-line
-    useEffect(() => {
-      if (adHocFilters.length > 0) {
-        onChange({ ...initializedQuery, adHocFilters: adHocFilters });
-      }
-
-      // eslint-disable-next-line
-    }, [adHocFilters.length]);
-  }
+  }, [props.app, adHocFilters.length]);
 
   return (
     <>
@@ -64,7 +60,6 @@ export function QueryEditor(props: QueryEditorProps<CHDataSource, CHQuery, CHDat
           adhocFilters={initializedQuery.adHocFilters}
           areAdHocFiltersAvailable={areAdHocFiltersAvailable}
           query={initializedQuery}
-          height={200}
           onSqlChange={onSqlChange}
           onRunQuery={onTriggerQuery}
           onFieldChange={onFieldChange}

src/views/QueryEditor/components/QueryBuilder/QueryBuilder.tsx

@@ -3,7 +3,7 @@ import { InlineField, InlineFieldRow, InlineLabel, Select } from '@grafana/ui';
 import { SelectableValue } from '@grafana/data';
 import { UniversalSelectField } from './components/UniversalSelectComponent';
 import { TimestampFormat } from '../../../../types/types';
-import { useConnectionData } from './components/useConnectionData';
+import { useConnectionData } from './hooks/useConnectionData';
 
 const options = [
   { label: 'DateTime', value: TimestampFormat.DateTime },

src/views/QueryEditor/components/QueryHeader/QueryHeader.constants.ts

@@ -0,0 +1,23 @@
+import {SelectableValue} from "@grafana/data";
+import {EditorMode} from "../../../../types/types";
+import {E2ESelectors} from "@grafana/e2e-selectors";
+
+const Components = {
+  QueryEditor: {
+    EditorMode: {
+      options: {
+        QuerySettings: 'Query Settings',
+        SQLEditor: 'SQL Editor',
+      },
+    },
+  },
+};
+
+const selectors: { components: E2ESelectors<typeof Components> } = {
+  components: Components,
+};
+
+export const QueryHeaderTabs: Array<SelectableValue<EditorMode>> = [
+  { label: selectors.components.QueryEditor.EditorMode.options.QuerySettings, value: EditorMode.Builder },
+  { label: selectors.components.QueryEditor.EditorMode.options.SQLEditor, value: EditorMode.SQL },
+];