New component: discovery.puppetdb

CHANGELOG.md

@@ -59,6 +59,8 @@ Main (unreleased)
   - `discovery.eureka` discovers targets from a Eureka Service Registry. (@spartan0x117)
   - `discovery.openstack` - service discovery for OpenStack. (@marctc)
   - `discovery.hetzner` - service discovery for Hetzner Cloud. (@marctc)
+  - `discovery.nomad` - service discovery from Nomad. (@captncraig)
+  - `discovery.puppetdb` - service discovery from PuppetDB. (@captncraig)
 
 ### Bugfixes
 

component/all/all.go

@@ -17,6 +17,7 @@ import (
 	_ "github.com/grafana/agent/component/discovery/kubernetes"                     // Import discovery.kubernetes
 	_ "github.com/grafana/agent/component/discovery/nomad"                          // Import discovery.nomad
 	_ "github.com/grafana/agent/component/discovery/openstack"                      // Import discovery.openstack
+	_ "github.com/grafana/agent/component/discovery/puppetdb"                       // Import discovery.puppetdb
 	_ "github.com/grafana/agent/component/discovery/relabel"                        // Import discovery.relabel
 	_ "github.com/grafana/agent/component/discovery/uyuni"                          // Import discovery.uyuni
 	_ "github.com/grafana/agent/component/local/file"                               // Import local.file

component/discovery/puppetdb/puppetdb.go

@@ -0,0 +1,81 @@
+package puppetdb
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/grafana/agent/component"
+	"github.com/grafana/agent/component/common/config"
+	"github.com/grafana/agent/component/discovery"
+	"github.com/prometheus/common/model"
+	prom_discovery "github.com/prometheus/prometheus/discovery/puppetdb"
+)
+
+func init() {
+	component.Register(component.Registration{
+		Name:    "discovery.puppetdb",
+		Args:    Arguments{},
+		Exports: discovery.Exports{},
+
+		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
+			return New(opts, args.(Arguments))
+		},
+	})
+}
+
+type Arguments struct {
+	HTTPClientConfig  config.HTTPClientConfig `river:",squash"`
+	RefreshInterval   time.Duration           `river:"refresh_interval,attr,optional"`
+	URL               string                  `river:"url,attr"`
+	Query             string                  `river:"query,attr"`
+	IncludeParameters bool                    `river:"include_parameters,attr,optional"`
+	Port              int                     `river:"port,attr,optional"`
+}
+
+var DefaultArguments = Arguments{
+	RefreshInterval:  60 * time.Second,
+	Port:             80,
+	HTTPClientConfig: config.DefaultHTTPClientConfig,
+}
+
+// SetToDefault implements river.Defaulter.
+func (args *Arguments) SetToDefault() {
+	*args = DefaultArguments
+}
+
+// Validate implements river.Validator.
+func (args *Arguments) Validate() error {
+	parsedURL, err := url.Parse(args.URL)
+	if err != nil {
+		return err
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return fmt.Errorf("URL scheme must be 'http' or 'https'")
+	}
+	if parsedURL.Host == "" {
+		return fmt.Errorf("host is missing in URL")
+	}
+	return args.HTTPClientConfig.Validate()
+}
+
+func (args *Arguments) Convert() *prom_discovery.SDConfig {
+	httpClient := &args.HTTPClientConfig
+
+	return &prom_discovery.SDConfig{
+		URL:               args.URL,
+		Query:             args.Query,
+		IncludeParameters: args.IncludeParameters,
+		Port:              args.Port,
+		RefreshInterval:   model.Duration(args.RefreshInterval),
+		HTTPClientConfig:  *httpClient.Convert(),
+	}
+}
+
+// New returns a new instance of a discovery.puppetdb component.
+func New(opts component.Options, args Arguments) (*discovery.Component, error) {
+	return discovery.New(opts, args, func(args component.Arguments) (discovery.Discoverer, error) {
+		newArgs := args.(Arguments)
+		return prom_discovery.NewDiscovery(newArgs.Convert(), opts.Logger)
+	})
+}

component/discovery/puppetdb/puppetdb_test.go

@@ -0,0 +1,68 @@
+package puppetdb
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/agent/pkg/river"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+	"gotest.tools/assert"
+)
+
+var exampleRiverConfig = `
+url = "https://www.example.com"
+query = "abc"
+include_parameters = true
+port = 29
+refresh_interval = "1m"
+basic_auth {
+	username = "123"
+	password = "456"
+}
+`
+
+func TestRiverConfig(t *testing.T) {
+	var args Arguments
+	err := river.Unmarshal([]byte(exampleRiverConfig), &args)
+	require.NoError(t, err)
+	assert.Equal(t, args.HTTPClientConfig.BasicAuth.Username, "123")
+	assert.Equal(t, args.RefreshInterval, time.Minute)
+	assert.Equal(t, args.URL, "https://www.example.com")
+	assert.Equal(t, args.Query, "abc")
+	assert.Equal(t, args.IncludeParameters, true)
+	assert.Equal(t, args.Port, 29)
+}
+
+func TestConvert(t *testing.T) {
+	var args Arguments
+	err := river.Unmarshal([]byte(exampleRiverConfig), &args)
+	require.NoError(t, err)
+
+	sd := args.Convert()
+	assert.Equal(t, "https://www.example.com", sd.URL)
+	assert.Equal(t, model.Duration(60*time.Second), sd.RefreshInterval)
+	assert.Equal(t, "abc", sd.Query)
+	assert.Equal(t, true, sd.IncludeParameters)
+	assert.Equal(t, 29, sd.Port)
+}
+
+func TestValidate(t *testing.T) {
+	riverArgsBadUrl := Arguments{
+		URL: string([]byte{0x7f}), // a control character to make url.Parse fail
+	}
+	err := riverArgsBadUrl.Validate()
+	assert.ErrorContains(t, err, "net/url: invalid")
+
+	riverArgsBadScheme := Arguments{
+		URL: "smtp://foo.bar",
+	}
+	err = riverArgsBadScheme.Validate()
+	assert.ErrorContains(t, err, "URL scheme must be")
+
+	riverArgsNoHost := Arguments{
+		URL: "http://#abc",
+	}
+	err = riverArgsNoHost.Validate()
+	assert.ErrorContains(t, err, "host is missing in URL")
+}

docs/sources/flow/reference/components/discovery.puppetdb.md

@@ -0,0 +1,152 @@
+---
+canonical: https://grafana.com/docs/agent/latest/flow/reference/components/discovery.puppetdb/
+title: discovery.puppetdb
+---
+
+# discovery.puppetdb
+
+`discovery.puppetdb` allows you to retrieve scrape targets from [PuppetDB](https://www.puppet.com/docs/puppetdb/7/overview.html) resources.
+
+This SD discovers resources and will create a target for each resource returned by the API.
+
+The resource address is the `certname` of the resource, and can be changed during relabeling.
+
+The queries for this component are expected to be valid [PQL (Puppet Query Language)](https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html).
+
+## Usage
+
+```river
+discovery.puppetdb "LABEL" {
+  url = PUPPET_SERVER
+}
+```
+
+## Arguments
+
+The following arguments are supported:
+
+Name | Type | Description | Default | Required
+---- | ---- | ----------- | ------- | --------
+`url` | `string` | The URL of the PuppetDB root query endpoint. | | yes
+`query` | `string` | Puppet Query Language (PQL) query. Only resources are supported. | | yes
+`include_parameters` | `bool` | Whether to include the parameters as meta labels. Due to the differences between parameter types and Prometheus labels, some parameters might not be rendered. The format of the parameters might also change in future releases. Make sure that you don't have secrets exposed as parameters if you enable this. | `false` | no
+`port` | `int` | The port to scrape metrics from.. | `80` | no
+`refresh_interval` | `duration` | Frequency to refresh targets. | `"30s"` | no
+`bearer_token` | `secret` | Bearer token to authenticate with. | | no
+`bearer_token_file` | `string` | File containing a bearer token to authenticate with. | | no
+`proxy_url` | `string` | HTTP proxy to proxy requests through. | | no
+`follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no
+`enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no
+
+ You can provide one of the following arguments for authentication:
+ - [`bearer_token` argument](#arguments).
+ - [`bearer_token_file` argument](#arguments). 
+ - [`basic_auth` block][basic_auth].
+ - [`authorization` block][authorization].
+ - [`oauth2` block][oauth2].
+
+[arguments]: #arguments
+
+## Blocks
+
+The following blocks are supported inside the definition of
+`discovery.puppetdb`:
+
+Hierarchy | Block | Description | Required
+--------- | ----- | ----------- | --------
+basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no
+authorization | [authorization][] | Configure generic authorization to the endpoint. | no
+oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no
+oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no
+
+The `>` symbol indicates deeper levels of nesting. For example,
+`oauth2 > tls_config` refers to a `tls_config` block defined inside
+an `oauth2` block.
+
+[basic_auth]: #basic_auth-block
+[authorization]: #authorization-block
+[oauth2]: #oauth2-block
+[tls_config]: #tls_config-block
+
+### basic_auth block
+
+{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" >}}
+
+### authorization block
+
+{{< docs/shared lookup="flow/reference/components/authorization-block.md" source="agent" >}}
+
+### oauth2 block
+
+{{< docs/shared lookup="flow/reference/components/oauth2-block.md" source="agent" >}}
+
+### tls_config block
+
+{{< docs/shared lookup="flow/reference/components/tls-config-block.md" source="agent" >}}
+
+## Exported fields
+
+The following fields are exported and can be referenced by other components:
+
+Name | Type | Description
+---- | ---- | -----------
+`targets` | `list(map(string))` | The set of targets discovered from puppetdb.
+
+Each target includes the following labels:
+
+* `__meta_puppetdb_query`: the Puppet Query Language (PQL) query.
+* `__meta_puppetdb_certname`: the name of the node associated with the resourcet.
+* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification.
+* `__meta_puppetdb_type`: the resource type.
+* `__meta_puppetdb_title`: the resource title.
+* `__meta_puppetdb_exported`: whether the resource is exported ("true" or "false").
+* `__meta_puppetdb_tags`: comma separated list of resource tags.
+* `__meta_puppetdb_file`: the manifest file in which the resource was declared.
+* `__meta_puppetdb_environment`: the environment of the node associated with the resource.
+* `__meta_puppetdb_parameter_<parametername>`: the parameters of the resource.
+
+## Component health
+
+`discovery.puppetdb` is only reported as unhealthy when given an invalid
+configuration. In those cases, exported fields retain their last healthy
+values.
+
+## Debug information
+
+`discovery.puppetdb` does not expose any component-specific debug information.
+
+### Debug metrics
+
+`discovery.puppetdb` does not expose any component-specific debug metrics.
+
+## Example
+
+This example discovers targets from puppetdb for all the servers that have a specific package defined:
+
+```river
+discovery.puppetdb "example" {
+	url   = "http://puppetdb.local:8080"
+	query = "resources { type = \"Package\" and title = \"node_exporter\" }"
+	port  = 9100
+}
+
+prometheus.scrape "demo" {
+	targets    = discovery.puppetdb.example.targets
+	forward_to = [prometheus.remote_write.demo.receiver]
+}
+
+prometheus.remote_write "demo" {
+	endpoint {
+		url = PROMETHEUS_REMOTE_WRITE_URL
+
+		basic_auth {
+			username = USERNAME
+			password = PASSWORD
+		}
+	}
+}
+```
+Replace the following:
+  - `PROMETHEUS_REMOTE_WRITE_URL`: The URL of the Prometheus remote_write-compatible server to send metrics to.
+  - `USERNAME`: The username to use for authentication to the remote_write API.
+  - `PASSWORD`: The password to use for authentication to the remote_write API.