Agent Management: Introduce support for template variables (#5788)

* Agent Management: Introduce support for template variables

  - This change allows managing template variables for remote
    configuration on a per-agent basis.
  - Both base configurations and snippets can be interpreted as
    templates and evaluated at load time with the provided template
    variables.
  - Templates must follow go's `text/template` syntax.
  - This greatly increases the flexibility and reusability of snippets.
  - Template evaluation has been tested in different scenarios and seems
    really robust. If the variables defined in the template cannot be
    resolved (even nested ones), and empty string is rendered
    instead.
  - Note: templates are only evaluated when the `template_variables`
    field within the `agent_metadata` remote config field is non-empty.
  - Note: this feature only applies to static mode.

* Improve naming

* Check error for template execution

* Add tests

  - Tests different scenarios, including:
    - Referencing non existing nested objects
    - Conditionals
    - Ranges
    - Character escaping

* Update CHANGELOG

* Always evaluate templates

  - This is required because certain agents might start before their
    labels are synced. If some of the snippets assigned to them contain
    template variables, loading the config will fail.

* Add test for template inside a template

  - Templates inside templates must be escaped using backticks to avoid
    them being evaluated by the snippet template execution

* Move feature to the next release in CHANGELOG

* Document templating functionality

* Fix doc

CHANGELOG.md

@@ -10,6 +10,10 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+### Features
+
+- Agent Management: Introduce support for templated configuration. (@jcreixell)
+
 ### Bugfixes
 
 - Permit `X-Faro-Session-ID` header in CORS requests for the `faro.receiver`

docs/sources/static/configuration/agent-management.md

@@ -123,7 +123,6 @@ selector:
 
 > **Note:** Snippet selection is currently done in the API server. This behaviour is subject to change in the future.
 
-
 ### Example response body
 
 ```yaml
@@ -164,3 +163,9 @@ snippets:
       os: linux
       app: app1
 ```
+
+> **Note:** Base configurations and snippets can contain go's [text/template](https://pkg.go.dev/text/template) actions. If you need preserve the literal value of a template action, you can escape it using backticks. For example:
+
+```
+{{ `{{ .template_var }}` }}
+```

pkg/config/agent_management_remote_config_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	process_exporter "github.com/grafana/agent/pkg/integrations/process_exporter"
 	"github.com/grafana/agent/pkg/metrics/instance"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
@@ -182,6 +183,82 @@ integration_configs:
 		require.Equal(t, 5*time.Second, c.Integrations.ConfigV1.IntegrationRestartBackoff)
 	})
 
+	t.Run("template variables provided", func(t *testing.T) {
+		baseConfig := `
+server:
+    log_level: {{.log_level}}
+`
+		templateInsideTemplate := "`{{ .template_inside_template }}`"
+		snippet := Snippet{
+			Config: `
+integration_configs:
+  process_exporter:
+    enabled: true
+    process_names:
+      - name: "grafana-agent"
+        cmdline:
+          - 'grafana-agent'
+      - name: "{{.nonexistent.foo.bar.baz.bat}}"
+        cmdline:
+          - "{{ ` + templateInsideTemplate + ` }}"
+      # Custom process monitors
+      {{- range $key, $value := .process_exporter_processes }}
+      - name: "{{ $value.name }}"
+        cmdline:
+          - "{{ $value.cmdline }}"
+        {{if $value.exe}}
+        exe:
+          - "{{ $value.exe }}"
+        {{end}}
+      {{- end }}
+`,
+		}
+
+		rc := RemoteConfig{
+			BaseConfig: BaseConfigContent(baseConfig),
+			Snippets:   []Snippet{snippet},
+			AgentMetadata: AgentMetadata{
+				TemplateVariables: map[string]any{
+					"log_level": "debug",
+					"process_exporter_processes": []map[string]string{
+						{
+							"name":    "java_processes",
+							"cmdline": ".*/java",
+						},
+						{
+							"name":    "{{.ExeFull}}:{{.Matches.Cfgfile}}",
+							"cmdline": `-config.path\\s+(?P<Cfgfile>\\S+)`,
+							"exe":     "/usr/local/bin/process-exporter",
+						},
+					},
+				},
+			},
+		}
+
+		c, err := rc.BuildAgentConfig()
+		require.NoError(t, err)
+		require.Equal(t, 1, len(c.Integrations.ConfigV1.Integrations))
+		processExporterConfig := c.Integrations.ConfigV1.Integrations[0].Config.(*process_exporter.Config)
+
+		require.Equal(t, 4, len(processExporterConfig.ProcessExporter))
+
+		require.Equal(t, "grafana-agent", processExporterConfig.ProcessExporter[0].Name)
+		require.Equal(t, "grafana-agent", processExporterConfig.ProcessExporter[0].CmdlineRules[0])
+		require.Equal(t, 0, len(processExporterConfig.ProcessExporter[0].ExeRules))
+
+		require.Equal(t, "<no value>", processExporterConfig.ProcessExporter[1].Name)
+		require.Equal(t, "{{ .template_inside_template }}", processExporterConfig.ProcessExporter[1].CmdlineRules[0])
+		require.Equal(t, 0, len(processExporterConfig.ProcessExporter[1].ExeRules))
+
+		require.Equal(t, "java_processes", processExporterConfig.ProcessExporter[2].Name)
+		require.Equal(t, ".*/java", processExporterConfig.ProcessExporter[2].CmdlineRules[0])
+		require.Equal(t, 0, len(processExporterConfig.ProcessExporter[2].ExeRules))
+
+		require.Equal(t, "{{.ExeFull}}:{{.Matches.Cfgfile}}", processExporterConfig.ProcessExporter[3].Name)
+		require.Equal(t, `-config.path\s+(?P<Cfgfile>\S+)`, processExporterConfig.ProcessExporter[3].CmdlineRules[0])
+		require.Equal(t, "/usr/local/bin/process-exporter", processExporterConfig.ProcessExporter[3].ExeRules[0])
+	})
+
 	t.Run("no external labels provided", func(t *testing.T) {
 		rc := RemoteConfig{
 			BaseConfig: BaseConfigContent(baseConfig),

pkg/config/agentmanagement_remote_config.go

@@ -1,6 +1,9 @@
 package config
 
 import (
+	"bytes"
+	"text/template"
+
 	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/logs"
 	"github.com/grafana/agent/pkg/metrics/instance"
@@ -28,7 +31,8 @@ type (
 	}
 
 	AgentMetadata struct {
-		ExternalLabels map[string]string `json:"external_labels,omitempty" yaml:"external_labels,omitempty"`
+		ExternalLabels    map[string]string `json:"external_labels,omitempty" yaml:"external_labels,omitempty"`
+		TemplateVariables map[string]any    `json:"template_variables,omitempty" yaml:"template_variables,omitempty"`
 	}
 
 	// SnippetContent defines the internal structure of a snippet configuration.
@@ -55,8 +59,13 @@ func NewRemoteConfig(buf []byte) (*RemoteConfig, error) {
 
 // BuildAgentConfig builds an agent configuration from a base config and a list of snippets
 func (rc *RemoteConfig) BuildAgentConfig() (*Config, error) {
+	baseConfig, err := evaluateTemplate(string(rc.BaseConfig), rc.AgentMetadata.TemplateVariables)
+	if err != nil {
+		return nil, err
+	}
+
 	c := DefaultConfig()
-	err := yaml.Unmarshal([]byte(rc.BaseConfig), &c)
+	err = yaml.Unmarshal([]byte(baseConfig), &c)
 	if err != nil {
 		return nil, err
 	}
@@ -66,15 +75,15 @@ func (rc *RemoteConfig) BuildAgentConfig() (*Config, error) {
 		return nil, err
 	}
 
-	err = appendSnippets(&c, rc.Snippets)
+	err = appendSnippets(&c, rc.Snippets, rc.AgentMetadata.TemplateVariables)
 	if err != nil {
 		return nil, err
 	}
 	appendExternalLabels(&c, rc.AgentMetadata.ExternalLabels)
 	return &c, nil
 }
 
-func appendSnippets(c *Config, snippets []Snippet) error {
+func appendSnippets(c *Config, snippets []Snippet, templateVars map[string]any) error {
 	metricsConfigs := instance.DefaultConfig
 	metricsConfigs.Name = "snippets"
 	logsConfigs := logs.InstanceConfig{
@@ -91,8 +100,13 @@ func appendSnippets(c *Config, snippets []Snippet) error {
 	}
 
 	for _, snippet := range snippets {
+		snippetConfig, err := evaluateTemplate(snippet.Config, templateVars)
+		if err != nil {
+			return err
+		}
+
 		var snippetContent SnippetContent
-		err := yaml.Unmarshal([]byte(snippet.Config), &snippetContent)
+		err = yaml.Unmarshal([]byte(snippetConfig), &snippetContent)
 		if err != nil {
 			return err
 		}
@@ -148,3 +162,18 @@ func appendExternalLabels(c *Config, externalLabels map[string]string) {
 		c.Logs.Global.ClientConfigs[i].ExternalLabels.LabelSet = logsExternalLabels.Merge(cc.ExternalLabels.LabelSet)
 	}
 }
+
+func evaluateTemplate(config string, templateVariables map[string]any) (string, error) {
+	tpl, err := template.New("config").Parse(config)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	err = tpl.Execute(&buf, templateVariables)
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}