diff --git a/component/prometheus/exporter/mssql/mssql.go b/component/prometheus/exporter/mssql/mssql.go index 819be78a8927..cbf8c7c58639 100644 --- a/component/prometheus/exporter/mssql/mssql.go +++ b/component/prometheus/exporter/mssql/mssql.go @@ -2,6 +2,7 @@ package mssql import ( "errors" + "os" "time" "github.com/grafana/agent/component" @@ -40,6 +41,7 @@ type Arguments struct { MaxIdleConnections int `river:"max_idle_connections,attr,optional"` MaxOpenConnections int `river:"max_open_connections,attr,optional"` Timeout time.Duration `river:"timeout,attr,optional"` + QueryConfigPath string `river:"query_config_path,attr,optional"` } // SetToDefault implements river.Defaulter. @@ -60,6 +62,21 @@ func (a *Arguments) Validate() error { if a.Timeout <= 0 { return errors.New("timeout must be positive") } + + if a.QueryConfigPath != "" { + _, err := os.Stat(a.QueryConfigPath) + + if err == nil { + return nil + } + + if errors.Is(err, os.ErrNotExist) { + return errors.New("query_config_path must be a valid path of a YAML config file") + } else { + return errors.New("query_config_path file has issues") + } + } + return nil } @@ -69,5 +86,6 @@ func (a *Arguments) Convert() *mssql.Config { MaxIdleConnections: a.MaxIdleConnections, MaxOpenConnections: a.MaxOpenConnections, Timeout: a.Timeout, + QueryConfigPath: a.QueryConfigPath, } } diff --git a/component/prometheus/exporter/mssql/mssql_test.go b/component/prometheus/exporter/mssql/mssql_test.go index b9a47ad3b776..48593d5a9ff2 100644 --- a/component/prometheus/exporter/mssql/mssql_test.go +++ b/component/prometheus/exporter/mssql/mssql_test.go @@ -1,6 +1,7 @@ package mssql import ( + "path/filepath" "testing" "time" @@ -12,12 +13,14 @@ import ( ) func TestRiverUnmarshal(t *testing.T) { + goodQueryPath, _ := filepath.Abs("../../../../pkg/integrations/mssql/collector_config.yaml") + riverConfig := ` connection_string = "sqlserver://user:pass@localhost:1433" max_idle_connections = 3 max_open_connections = 3 timeout = "10s" - ` + query_config_path = "` + goodQueryPath + `"` var args Arguments err := river.Unmarshal([]byte(riverConfig), &args) @@ -28,6 +31,7 @@ func TestRiverUnmarshal(t *testing.T) { MaxIdleConnections: 3, MaxOpenConnections: 3, Timeout: 10 * time.Second, + QueryConfigPath: goodQueryPath, } require.Equal(t, expected, args) @@ -47,6 +51,8 @@ func TestUnmarshalInvalid(t *testing.T) { } func TestArgumentsValidate(t *testing.T) { + goodQueryPath, _ := filepath.Abs("../../../../pkg/integrations/mssql/collector_config.yaml") + tests := []struct { name string args Arguments @@ -59,6 +65,7 @@ func TestArgumentsValidate(t *testing.T) { MaxIdleConnections: 1, MaxOpenConnections: 0, Timeout: 10 * time.Second, + QueryConfigPath: goodQueryPath, }, wantErr: true, }, @@ -69,6 +76,7 @@ func TestArgumentsValidate(t *testing.T) { MaxIdleConnections: 0, MaxOpenConnections: 1, Timeout: 10 * time.Second, + QueryConfigPath: goodQueryPath, }, wantErr: true, }, @@ -79,6 +87,18 @@ func TestArgumentsValidate(t *testing.T) { MaxIdleConnections: 1, MaxOpenConnections: 1, Timeout: 0, + QueryConfigPath: goodQueryPath, + }, + wantErr: true, + }, + { + name: "invalid query_config_path", + args: Arguments{ + ConnectionString: rivertypes.Secret("test"), + MaxIdleConnections: 1, + MaxOpenConnections: 1, + Timeout: 0, + QueryConfigPath: "doesnotexist.YAML", }, wantErr: true, }, @@ -89,6 +109,7 @@ func TestArgumentsValidate(t *testing.T) { MaxIdleConnections: 1, MaxOpenConnections: 1, Timeout: 10 * time.Second, + QueryConfigPath: goodQueryPath, }, wantErr: false, }, @@ -107,9 +128,10 @@ func TestArgumentsValidate(t *testing.T) { } func TestConvert(t *testing.T) { + goodQueryPath, _ := filepath.Abs("../../../../pkg/integrations/mssql/collector_config.yaml") riverConfig := ` connection_string = "sqlserver://user:pass@localhost:1433" - ` + query_config_path = "` + goodQueryPath + `"` var args Arguments err := river.Unmarshal([]byte(riverConfig), &args) require.NoError(t, err) @@ -121,6 +143,7 @@ func TestConvert(t *testing.T) { MaxIdleConnections: DefaultArguments.MaxIdleConnections, MaxOpenConnections: DefaultArguments.MaxOpenConnections, Timeout: DefaultArguments.Timeout, + QueryConfigPath: goodQueryPath, } require.Equal(t, expected, *res) } diff --git a/docs/sources/flow/reference/components/prometheus.exporter.mssql.md b/docs/sources/flow/reference/components/prometheus.exporter.mssql.md index 84786ee074a0..47cab2ababd6 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.mssql.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.mssql.md @@ -33,6 +33,7 @@ Omitted fields take their default values. | `max_idle_connections` | `int` | Maximum number of idle connections to any one target. | `3` | no | | `max_open_connections` | `int` | Maximum number of open connections to any one target. | `3` | no | | `timeout` | `duration` | The query timeout in seconds. | `"10s"` | no | +| `query_config_path` | `string` | The location of a custom query to prometheus metric config file. | | no | [The sql_exporter examples](https://github.com/burningalchemist/sql_exporter/blob/master/examples/azure-sql-mi/sql_exporter.yml#L21) show the format of the `connection_string` argument: @@ -100,3 +101,222 @@ Replace the following: - `PASSWORD`: The password to use for authentication to the remote_write API. [scrape]: {{< relref "./prometheus.scrape.md" >}} + +## Custom metrics +It is possible to retrieve custom prometheus metrics for a mssql instance using the optional `query_config_path` parameter. + +This parameter should point to a YAML config file defined [here](https://github.com/burningalchemist/sql_exporter#collectors). If it does, it will use the new config to query your mssql instance and create whatever metrics are defined. +If you want additional metrics on top of the default provided ones, the default config should be used as a base. + +The default config file used by this integration is as follows: +``` +collector_name: mssql_standard + +metrics: + - metric_name: mssql_local_time_seconds + type: gauge + help: 'Local time in seconds since epoch (Unix time).' + values: [unix_time] + query: | + SELECT DATEDIFF(second, '19700101', GETUTCDATE()) AS unix_time + - metric_name: mssql_connections + type: gauge + help: 'Number of active connections.' + key_labels: + - db + values: [count] + query: | + SELECT DB_NAME(sp.dbid) AS db, COUNT(sp.spid) AS count + FROM sys.sysprocesses sp + GROUP BY DB_NAME(sp.dbid) + # + # Collected from sys.dm_os_performance_counters + # + - metric_name: mssql_deadlocks_total + type: counter + help: 'Number of lock requests that resulted in a deadlock.' + values: [cntr_value] + query: | + SELECT cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Number of Deadlocks/sec' AND instance_name = '_Total' + - metric_name: mssql_user_errors_total + type: counter + help: 'Number of user errors.' + values: [cntr_value] + query: | + SELECT cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Errors/sec' AND instance_name = 'User Errors' + - metric_name: mssql_kill_connection_errors_total + type: counter + help: 'Number of severe errors that caused SQL Server to kill the connection.' + values: [cntr_value] + query: | + SELECT cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Errors/sec' AND instance_name = 'Kill Connection Errors' + - metric_name: mssql_page_life_expectancy_seconds + type: gauge + help: 'The minimum number of seconds a page will stay in the buffer pool on this node without references.' + values: [cntr_value] + query: | + SELECT top(1) cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Page life expectancy' + - metric_name: mssql_batch_requests_total + type: counter + help: 'Number of command batches received.' + values: [cntr_value] + query: | + SELECT cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Batch Requests/sec' + - metric_name: mssql_log_growths_total + type: counter + help: 'Number of times the transaction log has been expanded, per database.' + key_labels: + - db + values: [cntr_value] + query: | + SELECT rtrim(instance_name) AS db, cntr_value + FROM sys.dm_os_performance_counters WITH (NOLOCK) + WHERE counter_name = 'Log Growths' AND instance_name <> '_Total' + - metric_name: mssql_buffer_cache_hit_ratio + type: gauge + help: 'Ratio of requests that hit the buffer cache' + values: [BufferCacheHitRatio] + query: | + SELECT (a.cntr_value * 1.0 / b.cntr_value) * 100.0 as BufferCacheHitRatio + FROM sys.dm_os_performance_counters a + JOIN (SELECT cntr_value, OBJECT_NAME + FROM sys.dm_os_performance_counters + WHERE counter_name = 'Buffer cache hit ratio base' + AND OBJECT_NAME = 'SQLServer:Buffer Manager') b ON a.OBJECT_NAME = b.OBJECT_NAME + WHERE a.counter_name = 'Buffer cache hit ratio' + AND a.OBJECT_NAME = 'SQLServer:Buffer Manager' + + - metric_name: mssql_checkpoint_pages_sec + type: gauge + help: 'Checkpoint Pages Per Second' + values: [cntr_value] + query: | + SELECT cntr_value + FROM sys.dm_os_performance_counters + WHERE [counter_name] = 'Checkpoint pages/sec' + # + # Collected from sys.dm_io_virtual_file_stats + # + - metric_name: mssql_io_stall_seconds_total + type: counter + help: 'Stall time in seconds per database and I/O operation.' + key_labels: + - db + value_label: operation + values: + - read + - write + query_ref: mssql_io_stall + + # + # Collected from sys.dm_os_process_memory + # + - metric_name: mssql_resident_memory_bytes + type: gauge + help: 'SQL Server resident memory size (AKA working set).' + values: [resident_memory_bytes] + query_ref: mssql_process_memory + + - metric_name: mssql_virtual_memory_bytes + type: gauge + help: 'SQL Server committed virtual memory size.' + values: [virtual_memory_bytes] + query_ref: mssql_process_memory + + - metric_name: mssql_available_commit_memory_bytes + type: gauge + help: 'SQL Server available to be committed memory size.' + values: [available_commit_limit_bytes] + query_ref: mssql_process_memory + + - metric_name: mssql_memory_utilization_percentage + type: gauge + help: 'The percentage of committed memory that is in the working set.' + values: [memory_utilization_percentage] + query_ref: mssql_process_memory + + - metric_name: mssql_page_fault_count_total + type: counter + help: 'The number of page faults that were incurred by the SQL Server process.' + values: [page_fault_count] + query_ref: mssql_process_memory + + # + # Collected from sys.dm_os_sys_info + # + - metric_name: mssql_server_total_memory_bytes + type: gauge + help: 'SQL Server committed memory in the memory manager.' + values: [committed_memory_bytes] + query_ref: mssql_os_sys_info + + - metric_name: mssql_server_target_memory_bytes + type: gauge + help: 'SQL Server target committed memory set for the memory manager.' + values: [committed_memory_target_bytes] + query_ref: mssql_os_sys_info + + # + # Collected from sys.dm_os_sys_memory + # + - metric_name: mssql_os_memory + type: gauge + help: 'OS physical memory, used and available.' + value_label: 'state' + values: [used, available] + query: | + SELECT + (total_physical_memory_kb - available_physical_memory_kb) * 1024 AS used, + available_physical_memory_kb * 1024 AS available + FROM sys.dm_os_sys_memory + - metric_name: mssql_os_page_file + type: gauge + help: 'OS page file, used and available.' + value_label: 'state' + values: [used, available] + query: | + SELECT + (total_page_file_kb - available_page_file_kb) * 1024 AS used, + available_page_file_kb * 1024 AS available + FROM sys.dm_os_sys_memory +queries: + # Populates `mssql_io_stall` and `mssql_io_stall_total` + - query_name: mssql_io_stall + query: | + SELECT + cast(DB_Name(a.database_id) as varchar) AS [db], + sum(io_stall_read_ms) / 1000.0 AS [read], + sum(io_stall_write_ms) / 1000.0 AS [write] + FROM + sys.dm_io_virtual_file_stats(null, null) a + INNER JOIN sys.master_files b ON a.database_id = b.database_id AND a.file_id = b.file_id + GROUP BY a.database_id + # Populates `mssql_resident_memory_bytes`, `mssql_virtual_memory_bytes`, mssql_available_commit_memory_bytes, + # and `mssql_memory_utilization_percentage`, and `mssql_page_fault_count_total` + - query_name: mssql_process_memory + query: | + SELECT + physical_memory_in_use_kb * 1024 AS resident_memory_bytes, + virtual_address_space_committed_kb * 1024 AS virtual_memory_bytes, + available_commit_limit_kb * 1024 AS available_commit_limit_bytes, + memory_utilization_percentage, + page_fault_count + FROM sys.dm_os_process_memory + # Populates `mssql_server_total_memory_bytes` and `mssql_server_target_memory_bytes`. + - query_name: mssql_os_sys_info + query: | + SELECT + committed_kb * 1024 AS committed_memory_bytes, + committed_target_kb * 1024 AS committed_memory_target_bytes + FROM sys.dm_os_sys_info +``` diff --git a/pkg/integrations/mssql/sql_exporter.go b/pkg/integrations/mssql/sql_exporter.go index 94ea345bd57d..6efdfdcd47a8 100644 --- a/pkg/integrations/mssql/sql_exporter.go +++ b/pkg/integrations/mssql/sql_exporter.go @@ -72,7 +72,7 @@ func (c Config) validate() error { if errors.Is(err, os.ErrNotExist) { return errors.New("query_config_path must be a valid path of a YAML config file") } else { - return errors.New("query_config_path config not in correct format") + return errors.New("query_config_path file has issues") } } diff --git a/pkg/integrations/mssql/sql_exporter_test.go b/pkg/integrations/mssql/sql_exporter_test.go index 3b340c8f631e..f1184e74fd09 100644 --- a/pkg/integrations/mssql/sql_exporter_test.go +++ b/pkg/integrations/mssql/sql_exporter_test.go @@ -12,11 +12,8 @@ import ( ) func TestConfig_validate(t *testing.T) { - badQueryPath, err := filepath.Abs("./test/bad_query_config.yaml") - goodQueryPath, err := filepath.Abs("./collector_config.yaml") - if err != nil { + goodQueryPath, _ := filepath.Abs("./collector_config.yaml") - } testCases := []struct { name string input Config @@ -101,28 +98,6 @@ func TestConfig_validate(t *testing.T) { }, err: "query_config_path must be a valid path of a YAML config file", }, - { - name: "bad query config path", - input: Config{ - ConnectionString: "sqlserver://user:pass@localhost:1433", - MaxIdleConnections: 3, - MaxOpenConnections: 3, - Timeout: 10 * time.Second, - QueryConfigPath: "doesnotexist.YAML", - }, - err: "query_config_path must be a valid path of a YAML config file", - }, - { - name: "bad query config file", - input: Config{ - ConnectionString: "sqlserver://user:pass@localhost:1433", - MaxIdleConnections: 3, - MaxOpenConnections: 3, - Timeout: 10 * time.Second, - QueryConfigPath: badQueryPath, - }, - err: "query_config_path config not in correct format", - }, { name: "good query config file", input: Config{ @@ -148,6 +123,8 @@ func TestConfig_validate(t *testing.T) { } } func TestConfig_UnmarshalYaml(t *testing.T) { + goodQueryPath, _ := filepath.Abs("./collector_config.yaml") + t.Run("only required values", func(t *testing.T) { strConfig := `connection_string: "sqlserver://user:pass@localhost:1433"` @@ -169,7 +146,7 @@ connection_string: "sqlserver://user:pass@localhost:1433" max_idle_connections: 5 max_open_connections: 6 timeout: 1m -` +query_config_path: "` + goodQueryPath + `"` var c Config @@ -180,17 +157,22 @@ timeout: 1m MaxIdleConnections: 5, MaxOpenConnections: 6, Timeout: time.Minute, + QueryConfigPath: goodQueryPath, }, c) }) } func TestConfig_NewIntegration(t *testing.T) { + goodQueryPath, _ := filepath.Abs("./collector_config.yaml") + badQueryPath, _ := filepath.Abs("./test/bad_query_config.txt") + t.Run("integration with valid config", func(t *testing.T) { c := &Config{ ConnectionString: "sqlserver://user:pass@localhost:1433", MaxIdleConnections: 3, MaxOpenConnections: 3, Timeout: 10 * time.Second, + QueryConfigPath: goodQueryPath, } i, err := c.NewIntegration(log.NewJSONLogger(os.Stdout)) @@ -210,6 +192,20 @@ func TestConfig_NewIntegration(t *testing.T) { require.Nil(t, i) require.ErrorContains(t, err, "failed to validate config:") }) + + t.Run("integration with incorrect format for query config file", func(t *testing.T) { + c := &Config{ + ConnectionString: "sqlserver://user:pass@localhost:1433", + MaxIdleConnections: 3, + MaxOpenConnections: 3, + Timeout: 10 * time.Second, + QueryConfigPath: badQueryPath, + } + + i, err := c.NewIntegration(log.NewJSONLogger(os.Stdout)) + require.Nil(t, i) + require.ErrorContains(t, err, "failed to create mssql target: query_config_path file not in correct format: ") + }) } func TestConfig_AgentKey(t *testing.T) { diff --git a/pkg/integrations/mssql/test/bad_query_config.txt b/pkg/integrations/mssql/test/bad_query_config.txt new file mode 100644 index 000000000000..5f20a3621b56 --- /dev/null +++ b/pkg/integrations/mssql/test/bad_query_config.txt @@ -0,0 +1 @@ +This is not a valid config file. diff --git a/pkg/integrations/mssql/test/bad_query_config.yaml b/pkg/integrations/mssql/test/bad_query_config.yaml deleted file mode 100644 index 9f662452df8a..000000000000 --- a/pkg/integrations/mssql/test/bad_query_config.yaml +++ /dev/null @@ -1 +0,0 @@ -This is not a valid config file. \ No newline at end of file