feat(exporter/elasticsearch): add Basic Auth support (grafana#5814)

* feat(exporter/elasticsearch): add Basic Auth support

Signed-off-by: hainenber <[email protected]>

* misc: fix changelog entry (grafana#5812)

PR grafana#5802 accidentally added the changelog entry to a published release.

* fix(exporter/elasticsearch): only add Auth header when needed

Signed-off-by: hainenber <[email protected]>

* doc(exporter/elasticsearch): add Basic Auth block desc

Signed-off-by: hainenber <[email protected]>

* feat(exporter/elasticsearch): add Basic Auth support

Signed-off-by: hainenber <[email protected]>

* feat(exporter/elasticsearch): support Basic Auth for static config

Signed-off-by: hainenber <[email protected]>

* chore(CHANGELOG): remove merge conflict artifact

Signed-off-by: hainenber <[email protected]>

* misc: fix changelog entry

Signed-off-by: hainenber <[email protected]>

* feat(converter): map ES exporter's BasicAuth flow from static to flow

Signed-off-by: hainenber <[email protected]>

* fix(converter): nil check for ES exporter's Basic Auth

Signed-off-by: hainenber <[email protected]>

* lint

Signed-off-by: erikbaranowski <[email protected]>

---------

Signed-off-by: hainenber <[email protected]>
Signed-off-by: erikbaranowski <[email protected]>
Co-authored-by: Robert Fratto <[email protected]>
Co-authored-by: erikbaranowski <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -45,6 +45,8 @@ Main (unreleased)
   Previously, only `remote.*` and `local.*` components could be referenced
   without a circular dependency. (@rfratto)
 
+- Add support for Basic Auth-secured connection with Elasticsearch cluster using `prometheus.exporter.elasticsearch`. (@hainenber)
+
 - Add a `resource_to_telemetry_conversion` argument to `otelcol.exporter.prometheus`
   for converting resource attributes to Prometheus labels. (@hainenber)
 

component/prometheus/exporter/elasticsearch/elasticsearch.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/grafana/agent/component"
+	commonCfg "github.com/grafana/agent/component/common/config"
 	"github.com/grafana/agent/component/prometheus/exporter"
 	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
@@ -35,23 +36,24 @@ var DefaultArguments = Arguments{
 }
 
 type Arguments struct {
-	Address                   string        `river:"address,attr,optional"`
-	Timeout                   time.Duration `river:"timeout,attr,optional"`
-	AllNodes                  bool          `river:"all,attr,optional"`
-	Node                      string        `river:"node,attr,optional"`
-	ExportIndices             bool          `river:"indices,attr,optional"`
-	ExportIndicesSettings     bool          `river:"indices_settings,attr,optional"`
-	ExportClusterSettings     bool          `river:"cluster_settings,attr,optional"`
-	ExportShards              bool          `river:"shards,attr,optional"`
-	IncludeAliases            bool          `river:"aliases,attr,optional"`
-	ExportSnapshots           bool          `river:"snapshots,attr,optional"`
-	ExportClusterInfoInterval time.Duration `river:"clusterinfo_interval,attr,optional"`
-	CA                        string        `river:"ca,attr,optional"`
-	ClientPrivateKey          string        `river:"client_private_key,attr,optional"`
-	ClientCert                string        `river:"client_cert,attr,optional"`
-	InsecureSkipVerify        bool          `river:"ssl_skip_verify,attr,optional"`
-	ExportDataStreams         bool          `river:"data_stream,attr,optional"`
-	ExportSLM                 bool          `river:"slm,attr,optional"`
+	Address                   string               `river:"address,attr,optional"`
+	Timeout                   time.Duration        `river:"timeout,attr,optional"`
+	AllNodes                  bool                 `river:"all,attr,optional"`
+	Node                      string               `river:"node,attr,optional"`
+	ExportIndices             bool                 `river:"indices,attr,optional"`
+	ExportIndicesSettings     bool                 `river:"indices_settings,attr,optional"`
+	ExportClusterSettings     bool                 `river:"cluster_settings,attr,optional"`
+	ExportShards              bool                 `river:"shards,attr,optional"`
+	IncludeAliases            bool                 `river:"aliases,attr,optional"`
+	ExportSnapshots           bool                 `river:"snapshots,attr,optional"`
+	ExportClusterInfoInterval time.Duration        `river:"clusterinfo_interval,attr,optional"`
+	CA                        string               `river:"ca,attr,optional"`
+	ClientPrivateKey          string               `river:"client_private_key,attr,optional"`
+	ClientCert                string               `river:"client_cert,attr,optional"`
+	InsecureSkipVerify        bool                 `river:"ssl_skip_verify,attr,optional"`
+	ExportDataStreams         bool                 `river:"data_stream,attr,optional"`
+	ExportSLM                 bool                 `river:"slm,attr,optional"`
+	BasicAuth                 *commonCfg.BasicAuth `river:"basic_auth,block,optional"`
 }
 
 // SetToDefault implements river.Defaulter.
@@ -78,5 +80,6 @@ func (a *Arguments) Convert() *elasticsearch_exporter.Config {
 		InsecureSkipVerify:        a.InsecureSkipVerify,
 		ExportDataStreams:         a.ExportDataStreams,
 		ExportSLM:                 a.ExportSLM,
+		BasicAuth:                 a.BasicAuth.Convert(),
 	}
 }

component/prometheus/exporter/elasticsearch/elasticsearch_test.go

@@ -4,8 +4,11 @@ import (
 	"testing"
 	"time"
 
+	commonCfg "github.com/grafana/agent/component/common/config"
 	"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
 	"github.com/grafana/river"
+	"github.com/grafana/river/rivertypes"
+	promCfg "github.com/prometheus/common/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -27,6 +30,10 @@ func TestRiverUnmarshal(t *testing.T) {
 	ssl_skip_verify      = true
 	data_stream          = true
 	slm                  = true
+	basic_auth {
+		username = "username"
+		password = "pass"
+	}
 	`
 
 	var args Arguments
@@ -50,6 +57,10 @@ func TestRiverUnmarshal(t *testing.T) {
 		InsecureSkipVerify:        true,
 		ExportDataStreams:         true,
 		ExportSLM:                 true,
+		BasicAuth: &commonCfg.BasicAuth{
+			Username: "username",
+			Password: rivertypes.Secret("pass"),
+		},
 	}
 
 	require.Equal(t, expected, args)
@@ -73,6 +84,10 @@ func TestConvert(t *testing.T) {
 	ssl_skip_verify      = true
 	data_stream          = true
 	slm                  = true
+	basic_auth {
+		username = "username"
+		password = "pass"
+	}
 	`
 	var args Arguments
 	err := river.Unmarshal([]byte(riverConfig), &args)
@@ -97,6 +112,10 @@ func TestConvert(t *testing.T) {
 		InsecureSkipVerify:        true,
 		ExportDataStreams:         true,
 		ExportSLM:                 true,
+		BasicAuth: &promCfg.BasicAuth{
+			Username: "username",
+			Password: promCfg.Secret("pass"),
+		},
 	}
 	require.Equal(t, expected, *res)
 }

converter/internal/staticconvert/internal/build/elasticsearch_exporter.go

@@ -1,9 +1,11 @@
 package build
 
 import (
+	commonCfg "github.com/grafana/agent/component/common/config"
 	"github.com/grafana/agent/component/discovery"
 	"github.com/grafana/agent/component/prometheus/exporter/elasticsearch"
 	"github.com/grafana/agent/pkg/integrations/elasticsearch_exporter"
+	"github.com/grafana/river/rivertypes"
 )
 
 func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elasticsearch_exporter.Config, instanceKey *string) discovery.Exports {
@@ -12,7 +14,7 @@ func (b *IntegrationsConfigBuilder) appendElasticsearchExporter(config *elastics
 }
 
 func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsearch.Arguments {
-	return &elasticsearch.Arguments{
+	arg := &elasticsearch.Arguments{
 		Address:                   config.Address,
 		Timeout:                   config.Timeout,
 		AllNodes:                  config.AllNodes,
@@ -31,4 +33,14 @@ func toElasticsearchExporter(config *elasticsearch_exporter.Config) *elasticsear
 		ExportDataStreams:         config.ExportDataStreams,
 		ExportSLM:                 config.ExportSLM,
 	}
+
+	if config.BasicAuth != nil {
+		arg.BasicAuth = &commonCfg.BasicAuth{
+			Username:     config.BasicAuth.Username,
+			Password:     rivertypes.Secret(config.BasicAuth.Password),
+			PasswordFile: config.BasicAuth.PasswordFile,
+		}
+	}
+
+	return arg
 }

docs/sources/flow/reference/components/prometheus.exporter.elasticsearch.md

@@ -56,6 +56,21 @@ Omitted fields take their default values.
 | `data_streams`         | `bool`     | Export stats for Data Streams.                                                                         |                           | no       |
 | `slm`                  | `bool`     | Export stats for SLM (Snapshot Lifecycle Management).                                                  |                           | no       |
 
+## Blocks
+
+The following blocks are supported inside the definition of
+`prometheus.exporter.elasticsearch`:
+
+| Hierarchy           | Block             | Description                                              | Required |
+| ------------------- | ----------------- | -------------------------------------------------------- | -------- |
+| basic_auth          | [basic_auth][]    | Configure basic_auth for authenticating to the endpoint. | no       |
+
+[basic_auth]: #basic_auth-block
+
+### basic_auth block
+
+{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" version="<AGENT VERSION>" >}}
+
 ## Exported fields
 
 {{< docs/shared lookup="flow/reference/components/exporter-component-exports.md" source="agent" version="<AGENT_VERSION>" >}}
@@ -84,6 +99,10 @@ from `prometheus.exporter.elasticsearch`:
 ```river
 prometheus.exporter.elasticsearch "example" {
   address = "http://localhost:9200"
+  basic_auth {
+    username = USERNAME
+    password = PASSWORD
+  }
 }
 
 // Configure a prometheus.scrape component to collect Elasticsearch metrics.

docs/sources/static/configuration/integrations/elasticsearch-exporter-config.md

@@ -116,4 +116,12 @@ Full reference of options:
 
   # Export stats for SLM (Snapshot Lifecycle Management).
   [ slm: <boolean> ]
+
+  # Sets the `Authorization` header on every ES probe with the
+  # configured username and password.
+  # password and password_file are mutually exclusive.
+  basic_auth:
+    [ username: <string> ]
+    [ password: <secret> ]
+    [ password_file: <string> ]
 ```

pkg/integrations/elasticsearch_exporter/elasticsearch_exporter.go

@@ -4,9 +4,12 @@ package elasticsearch_exporter //nolint:golint
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/go-kit/log"
@@ -15,6 +18,7 @@ import (
 	integrations_v2 "github.com/grafana/agent/pkg/integrations/v2"
 	"github.com/grafana/agent/pkg/integrations/v2/metricsutils"
 	"github.com/prometheus/client_golang/prometheus"
+	promCfg "github.com/prometheus/common/config"
 
 	"github.com/prometheus-community/elasticsearch_exporter/collector"
 	"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
@@ -66,6 +70,21 @@ type Config struct {
 	ExportDataStreams bool `yaml:"data_stream,omitempty"`
 	// Export stats for Snapshot Lifecycle Management
 	ExportSLM bool `yaml:"slm,omitempty"`
+	// BasicAuth block allows secure connection with Elasticsearch cluster via Basic-Auth
+	BasicAuth *promCfg.BasicAuth `yaml:"basic_auth,omitempty"`
+}
+
+// Custom http.Transport struct for Basic Auth-secured communication with ES cluster
+type BasicAuthHTTPTransport struct {
+	http.Transport
+	authHeader string
+}
+
+func (b *BasicAuthHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if b.authHeader != "" {
+		req.Header.Add("authorization", b.authHeader)
+	}
+	return b.Transport.RoundTrip(req)
 }
 
 // UnmarshalYAML implements yaml.Unmarshaler for Config
@@ -115,14 +134,39 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) {
 	// returns nil if not provided and falls back to simple TCP.
 	tlsConfig := createTLSConfig(c.CA, c.ClientCert, c.ClientPrivateKey, c.InsecureSkipVerify)
 
-	httpClient := &http.Client{
-		Timeout: c.Timeout,
-		Transport: &http.Transport{
+	esHttpTransport := &BasicAuthHTTPTransport{
+		Transport: http.Transport{
 			TLSClientConfig: tlsConfig,
 			Proxy:           http.ProxyFromEnvironment,
 		},
 	}
 
+	if c.BasicAuth != nil {
+		password := string(c.BasicAuth.Password)
+		if len(c.BasicAuth.PasswordFile) > 0 {
+			buff, err := os.ReadFile(c.BasicAuth.PasswordFile)
+			if err != nil {
+				return nil, fmt.Errorf("unable to load password file %s: %w", c.BasicAuth.PasswordFile, err)
+			}
+			password = strings.TrimSpace(string(buff))
+		}
+		username := c.BasicAuth.Username
+		if len(c.BasicAuth.UsernameFile) > 0 {
+			buff, err := os.ReadFile(c.BasicAuth.UsernameFile)
+			if err != nil {
+				return nil, fmt.Errorf("unable to load username file %s: %w", c.BasicAuth.UsernameFile, err)
+			}
+			username = strings.TrimSpace(string(buff))
+		}
+		encodedAuth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
+		esHttpTransport.authHeader = "Basic " + encodedAuth
+	}
+
+	httpClient := &http.Client{
+		Timeout:   c.Timeout,
+		Transport: esHttpTransport,
+	}
+
 	clusterInfoRetriever := clusterinfo.New(logger, httpClient, esURL, c.ExportClusterInfoInterval)
 
 	collectors := []prometheus.Collector{