Add openstack config converter (#5208)

* add openstack config converter

* change tls_config from attr to block

* remove newline is diags

CHANGELOG.md

@@ -68,6 +68,8 @@ Main (unreleased)
 
 - Flow: improve river config validation step in `prometheus.scrape` by comparing `scrape_timeout` with `scrape_interval`. (@wildum)
 
+- Add `openstack` config converter to convert OpenStack yaml config (static mode) to river config (flow mode). (@wildum)
+
 
 ### Other changes
 

component/discovery/openstack/openstack.go

@@ -42,7 +42,7 @@ type Arguments struct {
 	RefreshInterval             time.Duration     `river:"refresh_interval,attr,optional"`
 	Port                        int               `river:"port,attr,optional"`
 	AllTenants                  bool              `river:"all_tenants,attr,optional"`
-	TLSConfig                   config.TLSConfig  `river:"tls_config,attr,optional"`
+	TLSConfig                   config.TLSConfig  `river:"tls_config,block,optional"`
 	Availability                string            `river:"availability,attr,optional"`
 }
 

component/discovery/openstack/openstack_test.go

@@ -4,8 +4,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/agent/component/common/config"
 	"github.com/grafana/river"
-	"github.com/prometheus/common/config"
+	promcfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/discovery/openstack"
 	"github.com/stretchr/testify/require"
@@ -28,6 +29,14 @@ func TestUnmarshal(t *testing.T) {
 	refresh_interval = "1m"
 	port = 80
 	all_tenants = true
+	tls_config {
+		ca_file = "/path/to/file.ca"
+		cert_file = "/path/to/file.cert"
+		key_file = "/path/to/file.key"
+		server_name = "server_name"
+		insecure_skip_verify = false
+		min_version = "TLS13"
+	}
 	`
 	var args Arguments
 	err := river.Unmarshal([]byte(cfg), &args)
@@ -72,13 +81,17 @@ func TestConvert(t *testing.T) {
 		Port:                      80,
 		AllTenants:                true,
 		Availability:              "public",
+		TLSConfig: config.TLSConfig{
+			Key:  "key",
+			Cert: "cert",
+		},
 	}
 	converted := args.Convert()
 
 	require.Equal(t, "http://openstack", converted.IdentityEndpoint)
 	require.Equal(t, "exampleuser", converted.Username)
 	require.Equal(t, "exampleuserid", converted.UserID)
-	require.Equal(t, config.Secret("examplepassword"), converted.Password)
+	require.Equal(t, promcfg.Secret("examplepassword"), converted.Password)
 	require.Equal(t, "exampleproject", converted.ProjectName)
 	require.Equal(t, "exampleprojectid", converted.ProjectID)
 	require.Equal(t, "exampledomain", converted.DomainName)
@@ -91,4 +104,6 @@ func TestConvert(t *testing.T) {
 	require.Equal(t, 80, converted.Port)
 	require.Equal(t, true, converted.AllTenants)
 	require.Equal(t, "public", converted.Availability)
+	require.Equal(t, promcfg.Secret("key"), converted.TLSConfig.Key)
+	require.Equal(t, "cert", converted.TLSConfig.Cert)
 }

converter/internal/prometheusconvert/openstack.go

@@ -0,0 +1,51 @@
+package prometheusconvert
+
+import (
+	"time"
+
+	"github.com/grafana/agent/component/discovery"
+	"github.com/grafana/agent/component/discovery/openstack"
+	"github.com/grafana/agent/converter/diag"
+	"github.com/grafana/agent/converter/internal/common"
+	"github.com/grafana/river/rivertypes"
+	prom_openstack "github.com/prometheus/prometheus/discovery/openstack"
+)
+
+func appendDiscoveryOpenstack(pb *prometheusBlocks, label string, sdConfig *prom_openstack.SDConfig) discovery.Exports {
+	discoveryOpenstackArgs := toDiscoveryOpenstack(sdConfig)
+	name := []string{"discovery", "openstack"}
+	block := common.NewBlockWithOverride(name, label, discoveryOpenstackArgs)
+	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
+	return NewDiscoveryExports("discovery.openstack." + label + ".targets")
+}
+
+func validateDiscoveryOpenstack(sdConfig *prom_openstack.SDConfig) diag.Diagnostics {
+	return nil
+}
+
+func toDiscoveryOpenstack(sdConfig *prom_openstack.SDConfig) *openstack.Arguments {
+	if sdConfig == nil {
+		return nil
+	}
+
+	return &openstack.Arguments{
+		IdentityEndpoint:            sdConfig.IdentityEndpoint,
+		Username:                    sdConfig.Username,
+		UserID:                      sdConfig.UserID,
+		Password:                    rivertypes.Secret(sdConfig.Password),
+		ProjectName:                 sdConfig.ProjectName,
+		ProjectID:                   sdConfig.ProjectID,
+		DomainName:                  sdConfig.DomainName,
+		DomainID:                    sdConfig.DomainID,
+		ApplicationCredentialName:   sdConfig.ApplicationCredentialName,
+		ApplicationCredentialID:     sdConfig.ApplicationCredentialID,
+		ApplicationCredentialSecret: rivertypes.Secret(sdConfig.ApplicationCredentialSecret),
+		Role:                        string(sdConfig.Role),
+		Region:                      sdConfig.Region,
+		RefreshInterval:             time.Duration(sdConfig.RefreshInterval),
+		Port:                        sdConfig.Port,
+		AllTenants:                  sdConfig.AllTenants,
+		TLSConfig:                   *ToTLSConfig(&sdConfig.TLSConfig),
+		Availability:                sdConfig.Availability,
+	}
+}

converter/internal/prometheusconvert/prometheusconvert.go

@@ -25,6 +25,7 @@ import (
 	prom_marathon "github.com/prometheus/prometheus/discovery/marathon"
 	prom_docker "github.com/prometheus/prometheus/discovery/moby"
 	prom_moby "github.com/prometheus/prometheus/discovery/moby"
+	prom_openstack "github.com/prometheus/prometheus/discovery/openstack"
 	prom_scaleway "github.com/prometheus/prometheus/discovery/scaleway"
 	prom_triton "github.com/prometheus/prometheus/discovery/triton"
 	prom_xds "github.com/prometheus/prometheus/discovery/xds"
@@ -189,6 +190,9 @@ func AppendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig
 		case *prom_nerve.NerveSDConfig:
 			labelCounts["nerve"]++
 			exports = appendDiscoveryNerve(pb, common.LabelWithIndex(labelCounts["nerve"]-1, label), sdc)
+		case *prom_openstack.SDConfig:
+			labelCounts["openstack"]++
+			exports = appendDiscoveryOpenstack(pb, common.LabelWithIndex(labelCounts["openstack"]-1, label), sdc)
 		case *prom_moby.DockerSwarmSDConfig:
 			labelCounts["dockerswarm"]++
 			exports = appendDiscoveryDockerswarm(pb, common.LabelWithIndex(labelCounts["dockerswarm"]-1, label), sdc)

converter/internal/prometheusconvert/testdata/openstack.river

@@ -0,0 +1,60 @@
+discovery.openstack "prometheus1" {
+	identity_endpoint             = "identity_endpoint"
+	username                      = "username"
+	userid                        = "userid"
+	password                      = "password"
+	project_name                  = "project_name"
+	project_id                    = "project_id"
+	domain_name                   = "domain_name"
+	domain_id                     = "domain_id"
+	application_credential_name   = "application_credential_name"
+	application_credential_id     = "application_credential_id"
+	application_credential_secret = "application_credential_secret"
+	role                          = "instance"
+	region                        = "region"
+	refresh_interval              = "20s"
+	port                          = 81
+	all_tenants                   = true
+
+	tls_config {
+		ca_pem               = "ca"
+		cert_pem             = "cert"
+		key_pem              = "key"
+		server_name          = "example.local"
+		insecure_skip_verify = true
+	}
+	availability = "internal"
+}
+
+discovery.openstack "prometheus2" {
+	role   = "hypervisor"
+	region = "region2"
+}
+
+prometheus.scrape "prometheus1" {
+	targets = concat(
+		discovery.openstack.prometheus1.targets,
+		[{
+			__address__ = "localhost:9090",
+		}],
+	)
+	forward_to = [prometheus.remote_write.default.receiver]
+	job_name   = "prometheus1"
+}
+
+prometheus.scrape "prometheus2" {
+	targets    = discovery.openstack.prometheus2.targets
+	forward_to = [prometheus.remote_write.default.receiver]
+	job_name   = "prometheus2"
+}
+
+prometheus.remote_write "default" {
+	endpoint {
+		name = "remote1"
+		url  = "http://remote-write-url1"
+
+		queue_config { }
+
+		metadata_config { }
+	}
+}

converter/internal/prometheusconvert/testdata/openstack.yaml

@@ -0,0 +1,37 @@
+scrape_configs:
+  - job_name: "prometheus1"
+    static_configs:
+      - targets: ["localhost:9090"]
+    openstack_sd_configs:
+      - identity_endpoint: "identity_endpoint"
+        username: "username"
+        userid: "userid"
+        password: "password"
+        project_name: "project_name"
+        project_id: "project_id"
+        domain_name: "domain_name"
+        domain_id: "domain_id"
+        application_credential_name: "application_credential_name"
+        application_credential_id: "application_credential_id"
+        application_credential_secret: "application_credential_secret"
+        role: "instance"
+        region: "region"
+        port: 81
+        all_tenants: true
+        availability: "internal"
+        refresh_interval: 20s
+        tls_config:
+          key: "key"
+          cert: "cert"
+          ca: "ca"
+          server_name: example.local
+          insecure_skip_verify: true
+
+  - job_name: "prometheus2"
+    openstack_sd_configs:
+      - role: "hypervisor"
+        region: "region2"
+
+remote_write:
+  - name: "remote1"
+    url: "http://remote-write-url1"

converter/internal/prometheusconvert/validate.go

@@ -22,6 +22,7 @@ import (
 	prom_marathon "github.com/prometheus/prometheus/discovery/marathon"
 	prom_docker "github.com/prometheus/prometheus/discovery/moby"
 	prom_moby "github.com/prometheus/prometheus/discovery/moby"
+	prom_openstack "github.com/prometheus/prometheus/discovery/openstack"
 	prom_scaleway "github.com/prometheus/prometheus/discovery/scaleway"
 	prom_triton "github.com/prometheus/prometheus/discovery/triton"
 	prom_kuma "github.com/prometheus/prometheus/discovery/xds"
@@ -126,6 +127,8 @@ func ValidateServiceDiscoveryConfigs(serviceDiscoveryConfigs prom_discover.Confi
 			diags.AddAll(validateDiscoveryServerset(sdc))
 		case *prom_nerve.NerveSDConfig:
 			diags.AddAll(validateDiscoveryNerve(sdc))
+		case *prom_openstack.SDConfig:
+			diags.AddAll(validateDiscoveryOpenstack(sdc))
 		case *prom_moby.DockerSwarmSDConfig:
 			diags.AddAll(validateDiscoveryDockerswarm(sdc))
 		default:

converter/internal/promtailconvert/testdata/openstack.river

@@ -0,0 +1,47 @@
+discovery.openstack "prometheus1" {
+	identity_endpoint             = "identity_endpoint"
+	username                      = "username"
+	userid                        = "userid"
+	password                      = "password"
+	project_name                  = "project_name"
+	project_id                    = "project_id"
+	domain_name                   = "domain_name"
+	domain_id                     = "domain_id"
+	application_credential_name   = "application_credential_name"
+	application_credential_id     = "application_credential_id"
+	application_credential_secret = "application_credential_secret"
+	role                          = "instance"
+	region                        = "region"
+	refresh_interval              = "20s"
+	port                          = 81
+	all_tenants                   = true
+
+	tls_config {
+		ca_pem               = "ca"
+		cert_pem             = "cert"
+		key_pem              = "key"
+		server_name          = "example.local"
+		insecure_skip_verify = true
+	}
+	availability = "internal"
+}
+
+discovery.openstack "prometheus1_2" {
+	role   = "hypervisor"
+	region = "region2"
+}
+
+local.file_match "prometheus1" {
+	path_targets = concat(
+		discovery.openstack.prometheus1.targets,
+		discovery.openstack.prometheus1_2.targets,
+		[{
+			__address__ = "localhost:9090",
+		}],
+	)
+}
+
+loki.source.file "prometheus1" {
+	targets    = local.file_match.prometheus1.targets
+	forward_to = []
+}

converter/internal/promtailconvert/testdata/openstack.yaml

@@ -0,0 +1,32 @@
+scrape_configs:
+  - job_name: "prometheus1"
+    static_configs:
+      - targets: ["localhost:9090"]
+    openstack_sd_configs:
+      - identity_endpoint: "identity_endpoint"
+        username: "username"
+        userid: "userid"
+        password: "password"
+        project_name: "project_name"
+        project_id: "project_id"
+        domain_name: "domain_name"
+        domain_id: "domain_id"
+        application_credential_name: "application_credential_name"
+        application_credential_id: "application_credential_id"
+        application_credential_secret: "application_credential_secret"
+        role: "instance"
+        region: "region"
+        port: 81
+        all_tenants: true
+        availability: "internal"
+        refresh_interval: 20s
+        tls_config:
+          key: "key"
+          cert: "cert"
+          ca: "ca"
+          server_name: example.local
+          insecure_skip_verify: true
+      - role: "hypervisor"
+        region: "region2"
+tracing: {enabled: false}
+server: {register_instrumentation: false}

converter/internal/promtailconvert/testdata/unsupported.diags

@@ -9,5 +9,4 @@
 (Warning) server.log_level is not supported - Flow mode components may produce different logs
 (Error) server.http_path_prefix is not supported
 (Warning) server.health_check_target disabling is not supported in Flow mode
-(Warning) stream_lag_labels is deprecated and the associated metric has been removed
-(Error) unsupported service discovery openstack was provided
+(Warning) stream_lag_labels is deprecated and the associated metric has been removed

converter/internal/promtailconvert/testdata/unsupported.yaml

@@ -25,9 +25,4 @@ server:
 
 scrape_configs:
   - job_name: unsupported
-    openstack_sd_configs:
-      - role: instance
-        domain_name: domain
-        region: region
-        port: 1234
     encoding: utf-16