Merge branch 'main' into switch-to-ibm-sarama

CHANGELOG.md

@@ -44,6 +44,8 @@ Main (unreleased)
 - Add a `file_watch` block in `loki.source.file` to configure how often to poll files from disk for changes via `min_poll_frequency` and `max_poll_frequency`.
   In static mode it can be configured in the global `file_watch_config` via `min_poll_frequency` and `max_poll_frequency`.  (@wildum)
 
+- Flow: In `prometheus.exporter.blackbox`, allow setting labels for individual targets. (@spartan0x117)
+
 ### Enhancements
 
 - Clustering: allow advertise interfaces to be configurable, with the possibility to select all available interfaces. (@wildum)
@@ -72,6 +74,10 @@ Main (unreleased)
   not consistent. (@rfratto)
 - Switch to `IBM/sarama` module. (@hainenber)
 
+### Bugfixes
+
+- Fixed a bug where `otelcol.processor.discovery` could modify the `targets` passed by an upstream component. (@ptodev)
+
 v0.36.1 (2023-09-06)
 --------------------
 

component/otelcol/processor/discovery/discovery.go

@@ -77,7 +77,6 @@ func (args *Arguments) Validate() error {
 
 // Component is the otelcol.exporter.discovery component.
 type Component struct {
-	cfg      Arguments
 	consumer *promsdconsumer.Consumer
 	logger   log.Logger
 }
@@ -134,18 +133,17 @@ func (c *Component) Run(ctx context.Context) error {
 // Update implements Component.
 func (c *Component) Update(newConfig component.Arguments) error {
 	cfg := newConfig.(Arguments)
-	c.cfg = cfg
 
 	hostLabels := make(map[string]discovery.Target)
 
-	for _, labels := range c.cfg.Targets {
+	for _, labels := range cfg.Targets {
 		host, err := promsdconsumer.GetHostFromLabels(labels)
 		if err != nil {
 			level.Warn(c.logger).Log("msg", "ignoring target, unable to find address", "err", err)
 			continue
 		}
-		promsdconsumer.CleanupLabels(labels)
-		hostLabels[host] = labels
+
+		hostLabels[host] = promsdconsumer.NewTargetsWithNonInternalLabels(labels)
 	}
 
 	err := c.consumer.UpdateOptions(promsdconsumer.Options{

component/prometheus/exporter/blackbox/blackbox.go

@@ -38,6 +38,10 @@ func buildBlackboxTargets(baseTarget discovery.Target, args component.Arguments)
 	a := args.(Arguments)
 	for _, tgt := range a.Targets {
 		target := make(discovery.Target)
+		// Set extra labels first, meaning that any other labels will override
+		for k, v := range tgt.Labels {
+			target[k] = v
+		}
 		for k, v := range baseTarget {
 			target[k] = v
 		}
@@ -62,9 +66,10 @@ var DefaultArguments = Arguments{
 
 // BlackboxTarget defines a target to be used by the exporter.
 type BlackboxTarget struct {
-	Name   string `river:",label"`
-	Target string `river:"address,attr"`
-	Module string `river:"module,attr,optional"`
+	Name   string            `river:",label"`
+	Target string            `river:"address,attr"`
+	Module string            `river:"module,attr,optional"`
+	Labels map[string]string `river:"labels,attr,optional"`
 }
 
 type TargetBlock []BlackboxTarget

component/prometheus/exporter/blackbox/blackbox_test.go

@@ -170,3 +170,47 @@ func TestBuildBlackboxTargets(t *testing.T) {
 	require.Equal(t, "http://example.com", targets[0]["__param_target"])
 	require.Equal(t, "http_2xx", targets[0]["__param_module"])
 }
+
+func TestBuildBlackboxTargetsWithExtraLabels(t *testing.T) {
+	baseArgs := Arguments{
+		ConfigFile: "modules.yml",
+		Targets: TargetBlock{{
+			Name:   "target_a",
+			Target: "http://example.com",
+			Module: "http_2xx",
+			Labels: map[string]string{
+				"env": "test",
+				"foo": "bar",
+			},
+		}},
+		ProbeTimeoutOffset: 1.0,
+	}
+	baseTarget := discovery.Target{
+		model.SchemeLabel:                   "http",
+		model.MetricsPathLabel:              "component/prometheus.exporter.blackbox.default/metrics",
+		"instance":                          "prometheus.exporter.blackbox.default",
+		"job":                               "integrations/blackbox",
+		"__meta_agent_integration_name":     "blackbox",
+		"__meta_agent_integration_instance": "prometheus.exporter.blackbox.default",
+	}
+	args := component.Arguments(baseArgs)
+	targets := buildBlackboxTargets(baseTarget, args)
+	require.Equal(t, 1, len(targets))
+	require.Equal(t, "integrations/blackbox/target_a", targets[0]["job"])
+	require.Equal(t, "http://example.com", targets[0]["__param_target"])
+	require.Equal(t, "http_2xx", targets[0]["__param_module"])
+
+	require.Equal(t, "test", targets[0]["env"])
+	require.Equal(t, "bar", targets[0]["foo"])
+
+	// Check that the extra labels do not override existing labels
+	baseArgs.Targets[0].Labels = map[string]string{
+		"job":      "test",
+		"instance": "test-instance",
+	}
+	args = component.Arguments(baseArgs)
+	targets = buildBlackboxTargets(baseTarget, args)
+	require.Equal(t, 1, len(targets))
+	require.Equal(t, "integrations/blackbox/target_a", targets[0]["job"])
+	require.Equal(t, "prometheus.exporter.blackbox.default", targets[0]["instance"])
+}

docs/Makefile

@@ -11,11 +11,11 @@ include docs.mk
 docs: check-cloudwatch-integration
 
 check-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.19.4-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.19.4-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
 
 generate-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.19.4-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate
 
 sources/assets/hierarchy.svg: sources/operator/hierarchy.dot
 	cat $< | $(PODMAN) run --rm -i nshine/dot dot -Tsvg > $@

docs/developer/windows/certificate_store/README.md

@@ -0,0 +1,143 @@
+# Guide to setting up a Windows Server for Certificate Testing
+
+This guide is used to set up a Windows Server for Windows Store Certificate Testing. This guide assumes you have downloaded a Windows 2022 Server ISO and have installed the Windows Server onto a virtual machine. Certificate Templates can only be installed on Windows Server and it must enabled as an Enterprise Certificate Authority. This guide is NOT meant to be a guide on how to set up a Windows Server for production use, and is meant only for setting up an environment to test the Grafana Agent and certificate store.
+
+## Prerequisites
+
+* The install should be fresh with no server roles defined or installed. 
+* You should be logged in via an administrator account.
+
+## Set up as domain controller
+
+1. Add `Active Directory Domain Services`.
+2. Add to a new forest, and when asked for a name, use `test.example.com`.
+3. Add a NETBIOS name `TESTCERT`.
+4. Continue the feature install until done.
+6. Reboot. If the set up succeeded, your username will be prefixed with `TESTCERT/` on login screen.
+
+
+## Set up certificate management
+
+For this setup we are using a one-node Domain Controller set up as the Enterprise CA.
+
+1. Go to the Server Management window.
+![](./images/initial.png)
+2. Click Add Roles.
+![](./images/addroles.png)
+3. Select `Active Directory Certificate Services`.
+![](./images/certificateservices.png)
+
+4. Select `Certificate Authority`, and `Certificate Authority Web Enrollment`. Note that we will need the `Certificate Web Enrollment Service`, but it must be installed after the `Certificate Authority`.
+
+5. Install and then select configure `Certificate Authority`.
+![](./images/configure_certificate_authority.png)
+
+6. Select `Enterprise CA`.
+![](./images/enterprise_ca.png)
+
+7. Select `Root CA`.
+![](./images/root_ca.png)
+
+8. Select `Create a new private key`.
+![](./images/private_key.png)
+
+9. Go with the defaults.
+![](./images/default_private_ky.png)
+
+10. Go with validity of 5 years.
+![](./images/validity.png)
+
+11. Install.
+12. When asked if you want to install additional services say yes.
+![](./images/additional_services.png)
+
+13. Select `Certificate Enrollment Web Service`.
+14. Use the default application pool identity.
+![](./images/default_identity.png)
+
+15. Choose existing SSL certificate.
+![](./images/default_certificate.png)
+
+16. Click through and complete.
+17. Reboot.
+
+## Creating a template
+
+1. From Server Management click Tools->Certificate Authority.
+![](./images/certificate_authority_click.png)
+
+2. Ensure `Certificate Templates` is available, if not then you are not set up as an Enterprise CA and go back to `Setup as domain controller`.
+![](./images/certificate_templates.png)
+
+3. Right-click on `Certificate Templates` and select `Manage`.
+![](./images/manage.png)
+
+4. This opens the `Certificate Templates Console`.
+5. Select `Authenticate Session` and `Duplicate`.
+6. Open `General` tab and change names to `Certstore Template`, set the Renewal Period to 32850 hours that is the max allowed.
+![](./images/template_general.png)
+
+7. Open `Cryptography` tab and select `Request can use any provider available on the subject's computer`.
+![](./images/availability.png)
+
+8. Open `Request Handling` tab and select `Allow private key to be exported`.
+![](./images/export_private_key.png)
+
+9. Open `Security` tab and give Administrator all the control.
+![](./images/give_control.png)
+
+10. Open `Extensions` tab click `Application Policies` click `Edit`, ensure `Any Purpose` is selected.
+![](./images/any_purpose.png)
+
+11. Close `Certificate Template Console`.
+12. Reopen `Certificate Authority` console.
+13. Right-click on `Certificate Templates`, select `New`, then `Certificate Template to Issue`.
+14. Select `Certstore Template`.
+15. It will now show in the Templates folder of Certificate Authority.
+
+## Creating a certificate from the template
+
+1. Open `certmgr` also called `Manage user certificates` from the Run button.
+2. Expand `Personal`, right-click on personal, select `All Tasks` , select `Request New Certificate`.
+3. Click through until you get the `Request Certificate` screen and select `Certstore Template`.
+![](./images/new_cert.png)
+
+4. Expand `Certstore Template` and select `Properties`.
+5. Give it a name  like `TestAuth` and ensure under `Private Key` -> `Key options` that the private key is exportable.
+![](./images/new_cert_exportable.png)
+
+6. Create Certificate.
+
+## Export the certificate
+
+1. From `certmgr` find the certificate you created and double click to open.
+2. Go to the `Details` tab and select `Copy to File...`.
+3. Ensure that `Allow private key to be exported` is selected.
+![](./images/export_private_key.png)
+
+4. Under `Export File Format` ensure that `Include all certificates in the certificate path if possible`.
+5. Export it to a file.
+
+## Setup Grafana Agent
+
+1. Open the Agent configuration file.
+2. Open `Certificate Templates Console`, right-click `Certstore Template` and find the Object identifier.
+![](./images/object_identifier.png)
+
+3. Copy the configuration file.
+4. Find the common name, it is `test-WIN-TF2GPD2VTOT-CA` in the screenshot.
+![](./images/common_name.png)
+
+5. Copy that to the configuration.
+6. Configuration should look like this.
+![](./images/config.png)
+
+7. Start Agent.
+
+## Copy certificate to browser
+
+1. Open browser.
+2. Find certificate management.
+3. Add certificate we exported in `Export the certificate`.
+4. Browse to `http://localhost:12345/metrics`, and it should work properly. You may need to tell it to trust the certificate though.
+

docs/sources/flow/reference/components/otelcol.processor.discovery.md

@@ -104,7 +104,7 @@ information.
 ## Examples
 
 ### Basic usage
-```
+```river
 discovery.http "dynamic_targets" {
     url              = "https://example.com/scrape_targets"
     refresh_interval = "15s"
@@ -123,7 +123,7 @@ otelcol.processor.discovery "default" {
 
 Outputs from more than one discovery process can be combined via the `concat` function.
 
-```
+```river
 discovery.http "dynamic_targets" {
     url              = "https://example.com/scrape_targets"
     refresh_interval = "15s"
@@ -149,7 +149,7 @@ It is not necessary to use a discovery component. In the example below, a `test_
 attribute will be added to a span if its IP address is "1.2.2.2". The `__internal_label__` will
 be not be added to the span, because it begins with a double underscore (`__`).
 
-```
+```river
 otelcol.processor.discovery "default" {
     targets = [{
         "__address__"        = "1.2.2.2", 

docs/sources/flow/reference/components/otelcol.processor.span.md

@@ -234,7 +234,7 @@ This example creates a new span name from the values of attributes `db.svc`,
 `operation`, and `id`, in that order, separated by the value `::`. 
 All attribute keys need to be specified in the span for the processor to rename it.
 
-```
+```river
 otelcol.processor.span "default" {
   name {
     separator        = "::"
@@ -247,20 +247,20 @@ otelcol.processor.span "default" {
 }
 ```
 
-For a span with the following attributes key/value pairs, the example
+For a span with the following attributes key/value pairs, the above
 Flow configuration will change the span name to `"location::get::1234"`:
-```
+```json
 { 
   "db.svc": "location", 
   "operation": "get", 
   "id": "1234"
 }
 ```
 
-For a span with the following attributes key/value pairs, the example 
+For a span with the following attributes key/value pairs, the above 
 Flow configuration will not change the span name. 
 This is because the attribute key `operation` isn't set:
-```
+```json
 { 
   "db.svc": "location", 
   "id": "1234"
@@ -269,7 +269,7 @@ This is because the attribute key `operation` isn't set:
 
 ### Creating a new span name from attribute values (no separator)
 
-```
+```river
 otelcol.processor.span "default" {
   name {
     from_attributes = ["db.svc", "operation", "id"]
@@ -281,9 +281,9 @@ otelcol.processor.span "default" {
 }
 ```
 
-For a span with the following attributes key/value pairs, the example
+For a span with the following attributes key/value pairs, the above
 Flow configuration will change the span name to `"locationget1234"`:
-```
+```json
 { 
   "db.svc": "location", 
   "operation": "get", 
@@ -298,7 +298,7 @@ Example input and output using the Flow configuration below:
 2. The span name will be changed to `/api/v1/document/{documentId}/update`
 3. A new attribute `"documentId"="12345678"` will be added to the span.
 
-```
+```river
 otelcol.processor.span "default" {
   name {
     to_attributes {
@@ -321,7 +321,7 @@ if the span has the following properties:
 - The span name contains `/` anywhere in the string.
 - The span name is not `donot/change`.
 
-```
+```river
 otelcol.processor.span "default" {
   include {
     match_type = "regexp"
@@ -348,7 +348,7 @@ otelcol.processor.span "default" {
 
 This example changes the status of a span to "Error" and sets an error description.
 
-```
+```river
 otelcol.processor.span "default" {
   status {
     code        = "Error"
@@ -366,7 +366,7 @@ otelcol.processor.span "default" {
 This example sets the status to success only when attribute `http.status_code` 
 is equal to `400`.
 
-```
+```river
 otelcol.processor.span "default" {
   include {
     match_type = "strict"