From 1f4037c28a3468829a0a5ed938755b5443e5ed86 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:36:05 -0400 Subject: [PATCH 1/9] discovery.puppetdb --- component/all/all.go | 1 + component/discovery/puppetdb/puppetdb.go | 81 +++++++++++++++++++ component/discovery/puppetdb/puppetdb_test.go | 49 +++++++++++ 3 files changed, 131 insertions(+) create mode 100644 component/discovery/puppetdb/puppetdb.go create mode 100644 component/discovery/puppetdb/puppetdb_test.go diff --git a/component/all/all.go b/component/all/all.go index 76425ef70d61..08d2fd7e32fb 100644 --- a/component/all/all.go +++ b/component/all/all.go @@ -13,6 +13,7 @@ import ( _ "github.com/grafana/agent/component/discovery/http" // Import discovery.http _ "github.com/grafana/agent/component/discovery/kubelet" // Import discovery.kubelet _ "github.com/grafana/agent/component/discovery/kubernetes" // Import discovery.kubernetes + _ "github.com/grafana/agent/component/discovery/puppetdb" // Import discovery.puppetdb _ "github.com/grafana/agent/component/discovery/relabel" // Import discovery.relabel _ "github.com/grafana/agent/component/local/file" // Import local.file _ "github.com/grafana/agent/component/local/file_match" // Import local.file_match diff --git a/component/discovery/puppetdb/puppetdb.go b/component/discovery/puppetdb/puppetdb.go new file mode 100644 index 000000000000..165a4c16a9c9 --- /dev/null +++ b/component/discovery/puppetdb/puppetdb.go @@ -0,0 +1,81 @@ +package puppetdb + +import ( + "fmt" + "net/url" + "time" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/common/config" + "github.com/grafana/agent/component/discovery" + "github.com/prometheus/common/model" + prom_discovery "github.com/prometheus/prometheus/discovery/puppetdb" +) + +func init() { + component.Register(component.Registration{ + Name: "discovery.puppetdb", + Args: Arguments{}, + Exports: discovery.Exports{}, + + Build: func(opts component.Options, args component.Arguments) (component.Component, error) { + return New(opts, args.(Arguments)) + }, + }) +} + +type Arguments struct { + HTTPClientConfig config.HTTPClientConfig `river:",squash"` + RefreshInterval time.Duration `river:"refresh_interval,attr,optional"` + URL string `river:"url,attr"` + Query string `river:"query,attr"` + IncludeParameters bool `river:"include_parameters,attr,optional"` + Port int `river:"port,attr,optional"` +} + +var DefaultArguments = Arguments{ + RefreshInterval: 60 * time.Second, + Port: 80, + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +// SetToDefault implements river.Defaulter. +func (args *Arguments) SetToDefault() { + *args = DefaultArguments +} + +// Validate implements river.Validator. +func (args *Arguments) Validate() error { + parsedURL, err := url.Parse(args.URL) + if err != nil { + return err + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return fmt.Errorf("URL scheme must be 'http' or 'https'") + } + if parsedURL.Host == "" { + return fmt.Errorf("host is missing in URL") + } + return args.HTTPClientConfig.Validate() +} + +func (args *Arguments) Convert() *prom_discovery.SDConfig { + httpClient := &args.HTTPClientConfig + + return &prom_discovery.SDConfig{ + URL: args.URL, + Query: args.Query, + IncludeParameters: args.IncludeParameters, + Port: args.Port, + RefreshInterval: model.Duration(args.RefreshInterval), + HTTPClientConfig: *httpClient.Convert(), + } +} + +// New returns a new instance of a discovery.consul component. +func New(opts component.Options, args Arguments) (*discovery.Component, error) { + return discovery.New(opts, args, func(args component.Arguments) (discovery.Discoverer, error) { + newArgs := args.(Arguments) + return prom_discovery.NewDiscovery(newArgs.Convert(), opts.Logger) + }) +} diff --git a/component/discovery/puppetdb/puppetdb_test.go b/component/discovery/puppetdb/puppetdb_test.go new file mode 100644 index 000000000000..79af45b0706e --- /dev/null +++ b/component/discovery/puppetdb/puppetdb_test.go @@ -0,0 +1,49 @@ +package puppetdb + +import ( + "testing" + "time" + + "github.com/grafana/agent/pkg/river" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "gotest.tools/assert" +) + +var exampleRiverConfig = ` +url = "https://www.example.com" +query = "abc" +include_parameters = true +port = 29 +refresh_interval = "1m" +basic_auth { + username = "123" + password = "456" +} +` + +func TestRiverConfig(t *testing.T) { + + var args Arguments + err := river.Unmarshal([]byte(exampleRiverConfig), &args) + require.NoError(t, err) + assert.Equal(t, args.HTTPClientConfig.BasicAuth.Username, "123") + assert.Equal(t, args.RefreshInterval, time.Minute) + assert.Equal(t, args.URL, "https://www.example.com") + assert.Equal(t, args.Query, "abc") + assert.Equal(t, args.IncludeParameters, true) + assert.Equal(t, args.Port, 29) +} + +func TestConvert(t *testing.T) { + var args Arguments + err := river.Unmarshal([]byte(exampleRiverConfig), &args) + require.NoError(t, err) + + sd := args.Convert() + assert.Equal(t, "https://www.example.com", sd.URL) + assert.Equal(t, model.Duration(60*time.Second), sd.RefreshInterval) + assert.Equal(t, "abc", sd.Query) + assert.Equal(t, true, sd.IncludeParameters) + assert.Equal(t, 29, sd.Port) +} From 8cae4fa316cc3bd206d0245c99e3155adcc0fb18 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:48:24 -0400 Subject: [PATCH 2/9] docs --- .../components/discovery.puppetdb.md | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/sources/flow/reference/components/discovery.puppetdb.md diff --git a/docs/sources/flow/reference/components/discovery.puppetdb.md b/docs/sources/flow/reference/components/discovery.puppetdb.md new file mode 100644 index 000000000000..aa636369cd45 --- /dev/null +++ b/docs/sources/flow/reference/components/discovery.puppetdb.md @@ -0,0 +1,153 @@ +--- +canonical: https://grafana.com/docs/agent/latest/flow/reference/components/discovery.puppetdb/ +title: discovery.puppetdb +--- + +# discovery.puppetdb + +`discovery.puppetdb` allows retrieving scrape targets from [PuppetDB](https://www.puppet.com/docs/puppetdb/7/overview.html) resources. + +This SD discovers resources and will create a target for each resource returned by the API. + +The resource address is the `certname` of the resource, and can be changed during relabeling. + +The queries for this component are expected to be valid [PQL (Puppet Query Language)](https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html). + +## Usage + +```river +discovery.puppetdb "LABEL" { + url = PUPPET_SERVER +} +``` + +## Arguments + +The following arguments are supported: + +Name | Type | Description | Default | Required +---- | ---- | ----------- | ------- | -------- +`url` | `string` | The URL of the PuppetDB root query endpoint. | | yes +`query` | `string` | Puppet Query Language (PQL) query. Only resources are supported. | | yes +`include_parameters` | `bool` | Whether to include the parameters as meta labels. Due to the differences between parameter types and Prometheus labels, some parameters might not be rendered. The format of the parameters might also change in future releases. Make sure that you don't have secrets exposed as parameters if you enable this. | `false` | no +`port` | `int` | The port to scrape metrics from.. | `80` | no + +`refresh_interval` | `duration` | Frequency to refresh targets. | `"30s"` | no +`bearer_token` | `secret` | Bearer token to authenticate with. | | no +`bearer_token_file` | `string` | File containing a bearer token to authenticate with. | | no +`proxy_url` | `string` | HTTP proxy to proxy requests through. | | no +`follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no +`enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no + + At most one of the following can be provided: + - [`bearer_token` argument](#arguments). + - [`bearer_token_file` argument](#arguments). + - [`basic_auth` block][basic_auth]. + - [`authorization` block][authorization]. + - [`oauth2` block][oauth2]. + +[arguments]: #arguments + +## Blocks + +The following blocks are supported inside the definition of +`discovery.puppetdb`: + +Hierarchy | Block | Description | Required +--------- | ----- | ----------- | -------- +basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no +authorization | [authorization][] | Configure generic authorization to the endpoint. | no +oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no +oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no + +The `>` symbol indicates deeper levels of nesting. For example, +`oauth2 > tls_config` refers to a `tls_config` block defined inside +an `oauth2` block. + +[basic_auth]: #basic_auth-block +[authorization]: #authorization-block +[oauth2]: #oauth2-block +[tls_config]: #tls_config-block + +### basic_auth block + +{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" >}} + +### authorization block + +{{< docs/shared lookup="flow/reference/components/authorization-block.md" source="agent" >}} + +### oauth2 block + +{{< docs/shared lookup="flow/reference/components/oauth2-block.md" source="agent" >}} + +### tls_config block + +{{< docs/shared lookup="flow/reference/components/tls-config-block.md" source="agent" >}} + +## Exported fields + +The following fields are exported and can be referenced by other components: + +Name | Type | Description +---- | ---- | ----------- +`targets` | `list(map(string))` | The set of targets discovered from puppetdb. + +Each target includes the following labels: + +* `__meta_puppetdb_query`: the Puppet Query Language (PQL) query. +* `__meta_puppetdb_certname`: the name of the node associated with the resourcet. +* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification. +* `__meta_puppetdb_type`: the resource type. +* `__meta_puppetdb_title`: the resource title. +* `__meta_puppetdb_exported`: whether the resource is exported ("true" or "false"). +* `__meta_puppetdb_tags`: comma separated list of resource tags. +* `__meta_puppetdb_file`: the manifest file in which the resource was declared. +* `__meta_puppetdb_environment`: the environment of the node associated with the resource. +* `__meta_puppetdb_parameter_`: the parameters of the resource. + +## Component health + +`discovery.puppetdb` is only reported as unhealthy when given an invalid +configuration. In those cases, exported fields retain their last healthy +values. + +## Debug information + +`discovery.puppetdb` does not expose any component-specific debug information. + +### Debug metrics + +`discovery.puppetdb` does not expose any component-specific debug metrics. + +## Example + +This example discovers targets from Consul for the specified list of services: + +```river +discovery.puppetdb "example" { + url = "http://puppetdb.local:8080" + query = "resources { type = \"Apache::Vhost\" }" + include_parameters = true +} + +prometheus.scrape "demo" { + targets = discovery.consul.example.targets + forward_to = [prometheus.remote_write.demo.receiver] +} + +prometheus.remote_write "demo" { + endpoint { + url = PROMETHEUS_REMOTE_WRITE_URL + + basic_auth { + username = USERNAME + password = PASSWORD + } + } +} +``` +Replace the following: + - `PROMETHEUS_REMOTE_WRITE_URL`: The URL of the Prometheus remote_write-compatible server to send metrics to. + - `USERNAME`: The username to use for authentication to the remote_write API. + - `PASSWORD`: The password to use for authentication to the remote_write API. \ No newline at end of file From fe606d2de8b198f5d4f75f249e37c54ea71b0568 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:51:57 -0400 Subject: [PATCH 3/9] fix example --- .../flow/reference/components/discovery.puppetdb.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sources/flow/reference/components/discovery.puppetdb.md b/docs/sources/flow/reference/components/discovery.puppetdb.md index aa636369cd45..5701ae001ebf 100644 --- a/docs/sources/flow/reference/components/discovery.puppetdb.md +++ b/docs/sources/flow/reference/components/discovery.puppetdb.md @@ -122,17 +122,17 @@ values. ## Example -This example discovers targets from Consul for the specified list of services: +This example discovers targets from puppetdb for all the servers that have a specific package defined: ```river discovery.puppetdb "example" { url = "http://puppetdb.local:8080" - query = "resources { type = \"Apache::Vhost\" }" - include_parameters = true + query = "resources { type = \"Package\" and title = \"node_exporter\" }" + port = 9100 } prometheus.scrape "demo" { - targets = discovery.consul.example.targets + targets = discovery.puppetdb.example.targets forward_to = [prometheus.remote_write.demo.receiver] } From c835cd61dd53fbcc680ea056affcece7b469481b Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:01:53 -0400 Subject: [PATCH 4/9] fix comment --- component/discovery/puppetdb/puppetdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/discovery/puppetdb/puppetdb.go b/component/discovery/puppetdb/puppetdb.go index 165a4c16a9c9..b1b05b0f930e 100644 --- a/component/discovery/puppetdb/puppetdb.go +++ b/component/discovery/puppetdb/puppetdb.go @@ -72,7 +72,7 @@ func (args *Arguments) Convert() *prom_discovery.SDConfig { } } -// New returns a new instance of a discovery.consul component. +// New returns a new instance of a discovery.puppetdb component. func New(opts component.Options, args Arguments) (*discovery.Component, error) { return discovery.New(opts, args, func(args component.Arguments) (discovery.Discoverer, error) { newArgs := args.(Arguments) From efbdbae9036ed311bb4ca6d1fb948633fbad7b76 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:40:05 -0400 Subject: [PATCH 5/9] Update docs/sources/flow/reference/components/discovery.puppetdb.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/flow/reference/components/discovery.puppetdb.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/discovery.puppetdb.md b/docs/sources/flow/reference/components/discovery.puppetdb.md index 5701ae001ebf..ee6404a09a42 100644 --- a/docs/sources/flow/reference/components/discovery.puppetdb.md +++ b/docs/sources/flow/reference/components/discovery.puppetdb.md @@ -5,7 +5,7 @@ title: discovery.puppetdb # discovery.puppetdb -`discovery.puppetdb` allows retrieving scrape targets from [PuppetDB](https://www.puppet.com/docs/puppetdb/7/overview.html) resources. +`discovery.puppetdb` allows you to retrieve scrape targets from [PuppetDB](https://www.puppet.com/docs/puppetdb/7/overview.html) resources. This SD discovers resources and will create a target for each resource returned by the API. From c7fba4defd657485492287b9ad0221c22b9b2d6b Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:40:12 -0400 Subject: [PATCH 6/9] Update docs/sources/flow/reference/components/discovery.puppetdb.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/flow/reference/components/discovery.puppetdb.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/discovery.puppetdb.md b/docs/sources/flow/reference/components/discovery.puppetdb.md index ee6404a09a42..621409840457 100644 --- a/docs/sources/flow/reference/components/discovery.puppetdb.md +++ b/docs/sources/flow/reference/components/discovery.puppetdb.md @@ -39,7 +39,7 @@ Name | Type | Description | Default | Required `follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no `enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no - At most one of the following can be provided: + You can provide one of the following arguments for authentication: - [`bearer_token` argument](#arguments). - [`bearer_token_file` argument](#arguments). - [`basic_auth` block][basic_auth]. From 3b98557d9de87f8570462127b2053823c8505646 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Tue, 8 Aug 2023 09:27:50 -0400 Subject: [PATCH 7/9] add test for validate --- component/discovery/puppetdb/puppetdb_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/component/discovery/puppetdb/puppetdb_test.go b/component/discovery/puppetdb/puppetdb_test.go index 79af45b0706e..437be8a26426 100644 --- a/component/discovery/puppetdb/puppetdb_test.go +++ b/component/discovery/puppetdb/puppetdb_test.go @@ -47,3 +47,23 @@ func TestConvert(t *testing.T) { assert.Equal(t, true, sd.IncludeParameters) assert.Equal(t, 29, sd.Port) } + +func TestValidate(t *testing.T) { + riverArgsBadUrl := Arguments{ + URL: string([]byte{0x7f}), // a control character to make url.Parse fail + } + err := riverArgsBadUrl.Validate() + assert.ErrorContains(t, err, "net/url: invalid") + + riverArgsBadScheme := Arguments{ + URL: "smtp://foo.bar", + } + err = riverArgsBadScheme.Validate() + assert.ErrorContains(t, err, "URL scheme must be") + + riverArgsNoHost := Arguments{ + URL: "http://#abc", + } + err = riverArgsNoHost.Validate() + assert.ErrorContains(t, err, "host is missing in URL") +} From 0444e502d986a9e5cbb8e2dad359e7af9854e933 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:57:45 -0400 Subject: [PATCH 8/9] Update docs/sources/flow/reference/components/discovery.puppetdb.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marc Tudurí --- docs/sources/flow/reference/components/discovery.puppetdb.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sources/flow/reference/components/discovery.puppetdb.md b/docs/sources/flow/reference/components/discovery.puppetdb.md index d4d9fef23c1f..a836bc805639 100644 --- a/docs/sources/flow/reference/components/discovery.puppetdb.md +++ b/docs/sources/flow/reference/components/discovery.puppetdb.md @@ -31,7 +31,6 @@ Name | Type | Description | Default | Required `query` | `string` | Puppet Query Language (PQL) query. Only resources are supported. | | yes `include_parameters` | `bool` | Whether to include the parameters as meta labels. Due to the differences between parameter types and Prometheus labels, some parameters might not be rendered. The format of the parameters might also change in future releases. Make sure that you don't have secrets exposed as parameters if you enable this. | `false` | no `port` | `int` | The port to scrape metrics from.. | `80` | no - `refresh_interval` | `duration` | Frequency to refresh targets. | `"30s"` | no `bearer_token` | `secret` | Bearer token to authenticate with. | | no `bearer_token_file` | `string` | File containing a bearer token to authenticate with. | | no From 5cb3c2dd85c43489abb1b10db401f914d28e1627 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Tue, 8 Aug 2023 11:58:42 -0400 Subject: [PATCH 9/9] lint --- component/discovery/puppetdb/puppetdb_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/component/discovery/puppetdb/puppetdb_test.go b/component/discovery/puppetdb/puppetdb_test.go index 437be8a26426..fb037be64888 100644 --- a/component/discovery/puppetdb/puppetdb_test.go +++ b/component/discovery/puppetdb/puppetdb_test.go @@ -23,7 +23,6 @@ basic_auth { ` func TestRiverConfig(t *testing.T) { - var args Arguments err := river.Unmarshal([]byte(exampleRiverConfig), &args) require.NoError(t, err)