From a6191b2dc299112687afb27ac9b0b3984dbc8021 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Tue, 14 Nov 2023 09:47:35 +0000 Subject: [PATCH] Add a otelcol.processor.resourcedetection component --- component/otelcol/config_k8s.go | 25 + .../internal/aws/ec2/config.go | 50 + .../internal/aws/ecs/config.go | 50 + .../internal/aws/eks/config.go | 26 + .../internal/aws/elasticbeanstalk/config.go | 32 + .../internal/aws/lambda/config.go | 40 + .../internal/azure/aks/config.go | 26 + .../internal/azure/config.go | 42 + .../internal/consul/config.go | 89 ++ .../internal/docker/config.go | 26 + .../resourcedetection/internal/gcp/config.go | 56 ++ .../internal/heroku/config.go | 38 + .../internal/k8snode/config.go | 65 ++ .../internal/openshift/config.go | 48 + .../resource_attribute_config.go | 28 + .../internal/system/config.go | 76 ++ .../resourcedetection/resourcedetection.go | 204 ++++ .../resourcedetection_test.go | 922 ++++++++++++++++++ .../otelcol.processor.resourcedetection.md | 261 +++++ go.mod | 6 + go.sum | 10 + 21 files changed, 2120 insertions(+) create mode 100644 component/otelcol/config_k8s.go create mode 100644 component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/aws/eks/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/azure/aks/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/azure/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/consul/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/docker/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/gcp/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/heroku/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/k8snode/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/openshift/config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/resource_attribute_config/resource_attribute_config.go create mode 100644 component/otelcol/processor/resourcedetection/internal/system/config.go create mode 100644 component/otelcol/processor/resourcedetection/resourcedetection.go create mode 100644 component/otelcol/processor/resourcedetection/resourcedetection_test.go create mode 100644 docs/sources/flow/reference/components/otelcol.processor.resourcedetection.md diff --git a/component/otelcol/config_k8s.go b/component/otelcol/config_k8s.go new file mode 100644 index 000000000000..978a678317d9 --- /dev/null +++ b/component/otelcol/config_k8s.go @@ -0,0 +1,25 @@ +package otelcol + +import "fmt" + +// KubernetesAPIConfig contains options relevant to connecting to the K8s API +type KubernetesAPIConfig struct { + // How to authenticate to the K8s API server. This can be one of `none` + // (for no auth), `serviceAccount` (to use the standard service account + // token provided to the agent pod), or `kubeConfig` to use credentials + // from `~/.kube/config`. + AuthType string `river:"auth_type,attr,optional"` + + // When using auth_type `kubeConfig`, override the current context. + Context string `river:"context,attr,optional"` +} + +// Validate returns an error if the config is invalid. +func (c *KubernetesAPIConfig) Validate() error { + switch c.AuthType { + case "none", "serviceAccount", "kubeConfig", "tls": + return nil + default: + return fmt.Errorf("invalid auth_type %q", c.AuthType) + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go b/component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go new file mode 100644 index 000000000000..bed0d0bfdf97 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go @@ -0,0 +1,50 @@ +package ec2 + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +// Config defines user-specified configurations unique to the EC2 detector +type Config struct { + // Tags is a list of regex's to match ec2 instance tag keys that users want + // to add as resource attributes to processed data + Tags []string `river:"tags,attr,optional"` + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + var tags []string + if args.Tags != nil { + tags = append([]string{}, args.Tags...) + } + + return map[string]interface{}{ + "tags": tags, + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/ec2 resource attributes. +type ResourceAttributesConfig struct { + CloudAccountID *rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"` + CloudAvailabilityZone *rac.ResourceAttributeConfig `river:"cloud.availability_zone,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + HostID *rac.ResourceAttributeConfig `river:"host.id,block,optional"` + HostImageID *rac.ResourceAttributeConfig `river:"host.image.id,block,optional"` + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` + HostType *rac.ResourceAttributeConfig `river:"host.type,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.account.id": r.CloudAccountID.Convert(), + "cloud.availability_zone": r.CloudAvailabilityZone.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "host.id": r.HostID.Convert(), + "host.image.id": r.HostImageID.Convert(), + "host.name": r.HostName.Convert(), + "host.type": r.HostType.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go b/component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go new file mode 100644 index 000000000000..87f46cd2b7ed --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go @@ -0,0 +1,50 @@ +package ecs + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/ecs resource attributes. +type ResourceAttributesConfig struct { + AwsEcsClusterArn *rac.ResourceAttributeConfig `river:"aws.ecs.cluster.arn,block,optional"` + AwsEcsLaunchtype *rac.ResourceAttributeConfig `river:"aws.ecs.launchtype,block,optional"` + AwsEcsTaskArn *rac.ResourceAttributeConfig `river:"aws.ecs.task.arn,block,optional"` + AwsEcsTaskFamily *rac.ResourceAttributeConfig `river:"aws.ecs.task.family,block,optional"` + AwsEcsTaskRevision *rac.ResourceAttributeConfig `river:"aws.ecs.task.revision,block,optional"` + AwsLogGroupArns *rac.ResourceAttributeConfig `river:"aws.log.group.arns,block,optional"` + AwsLogGroupNames *rac.ResourceAttributeConfig `river:"aws.log.group.names,block,optional"` + AwsLogStreamArns *rac.ResourceAttributeConfig `river:"aws.log.stream.arns,block,optional"` + AwsLogStreamNames *rac.ResourceAttributeConfig `river:"aws.log.stream.names,block,optional"` + CloudAccountID *rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"` + CloudAvailabilityZone *rac.ResourceAttributeConfig `river:"cloud.availability_zone,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "aws.ecs.cluster.arn": r.AwsEcsClusterArn.Convert(), + "aws.ecs.launchtype": r.AwsEcsLaunchtype.Convert(), + "aws.ecs.task.arn": r.AwsEcsTaskArn.Convert(), + "aws.ecs.task.family": r.AwsEcsTaskFamily.Convert(), + "aws.ecs.task.revision": r.AwsEcsTaskRevision.Convert(), + "aws.log.group.arns": r.AwsLogGroupArns.Convert(), + "aws.log.group.names": r.AwsLogGroupNames.Convert(), + "aws.log.stream.arns": r.AwsLogStreamArns.Convert(), + "aws.log.stream.names": r.AwsLogStreamNames.Convert(), + "cloud.account.id": r.CloudAccountID.Convert(), + "cloud.availability_zone": r.CloudAvailabilityZone.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/aws/eks/config.go b/component/otelcol/processor/resourcedetection/internal/aws/eks/config.go new file mode 100644 index 000000000000..41451ee2c9a4 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/aws/eks/config.go @@ -0,0 +1,26 @@ +package eks + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/eks resource attributes. +type ResourceAttributesConfig struct { + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go b/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go new file mode 100644 index 000000000000..7d863afe0431 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go @@ -0,0 +1,32 @@ +package elasticbeanstalk + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/elastic_beanstalk resource attributes. +type ResourceAttributesConfig struct { + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + DeploymentEnvironment *rac.ResourceAttributeConfig `river:"deployment.environment,block,optional"` + ServiceInstanceID *rac.ResourceAttributeConfig `river:"service.instance.id,block,optional"` + ServiceVersion *rac.ResourceAttributeConfig `river:"service.version,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "deployment.environment": r.DeploymentEnvironment.Convert(), + "service.instance.id": r.ServiceInstanceID.Convert(), + "service.version": r.ServiceVersion.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go b/component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go new file mode 100644 index 000000000000..ba0eb2a942f4 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go @@ -0,0 +1,40 @@ +package lambda + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/lambda resource attributes. +type ResourceAttributesConfig struct { + AwsLogGroupNames *rac.ResourceAttributeConfig `river:"aws.log.group.names,block,optional"` + AwsLogStreamNames *rac.ResourceAttributeConfig `river:"aws.log.stream.names,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + FaasInstance *rac.ResourceAttributeConfig `river:"faas.instance,block,optional"` + FaasMaxMemory *rac.ResourceAttributeConfig `river:"faas.max_memory,block,optional"` + FaasName *rac.ResourceAttributeConfig `river:"faas.name,block,optional"` + FaasVersion *rac.ResourceAttributeConfig `river:"faas.version,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "aws.log.group.names": r.AwsLogGroupNames.Convert(), + "aws.log.stream.names": r.AwsLogStreamNames.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "faas.instance": r.FaasInstance.Convert(), + "faas.max_memory": r.FaasMaxMemory.Convert(), + "faas.name": r.FaasName.Convert(), + "faas.version": r.FaasVersion.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/azure/aks/config.go b/component/otelcol/processor/resourcedetection/internal/azure/aks/config.go new file mode 100644 index 000000000000..6e0a37eff15d --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/azure/aks/config.go @@ -0,0 +1,26 @@ +package aks + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/aks resource attributes. +type ResourceAttributesConfig struct { + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/azure/config.go b/component/otelcol/processor/resourcedetection/internal/azure/config.go new file mode 100644 index 000000000000..19282417f711 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/azure/config.go @@ -0,0 +1,42 @@ +package azure + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/azure resource attributes. +type ResourceAttributesConfig struct { + AzureResourcegroupName *rac.ResourceAttributeConfig `river:"azure.resourcegroup.name,block,optional"` + AzureVMName *rac.ResourceAttributeConfig `river:"azure.vm.name,block,optional"` + AzureVMScalesetName *rac.ResourceAttributeConfig `river:"azure.vm.scaleset.name,block,optional"` + AzureVMSize *rac.ResourceAttributeConfig `river:"azure.vm.size,block,optional"` + CloudAccountID *rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + HostID *rac.ResourceAttributeConfig `river:"host.id,block,optional"` + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "azure.resourcegroup.name": r.AzureResourcegroupName.Convert(), + "azure.vm.name": r.AzureVMName.Convert(), + "azure.vm.scaleset.name": r.AzureVMScalesetName.Convert(), + "azure.vm.size": r.AzureVMSize.Convert(), + "cloud.account.id": r.CloudAccountID.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "host.id": r.HostID.Convert(), + "host.name": r.HostName.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/consul/config.go b/component/otelcol/processor/resourcedetection/internal/consul/config.go new file mode 100644 index 000000000000..07b4d560bb1e --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/consul/config.go @@ -0,0 +1,89 @@ +package consul + +import ( + rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + "github.com/grafana/river/rivertypes" + "go.opentelemetry.io/collector/config/configopaque" +) + +// The struct requires no user-specified fields by default as consul agent's default +// configuration will be provided to the API client. +// See `consul.go#NewDetector` for more information. +type Config struct { + // Address is the address of the Consul server + Address string `river:"address,attr,optional"` + + // Datacenter to use. If not provided, the default agent datacenter is used. + Datacenter string `river:"datacenter,attr,optional"` + + // Token is used to provide a per-request ACL token which overrides the + // agent's default (empty) token. Token is only required if + // [Consul's ACL System](https://www.consul.io/docs/security/acl/acl-system) + // is enabled. + Token rivertypes.Secret `river:"token,attr,optional"` + + // TokenFile is not necessary in River because users can use the local.file + // Flow component instead. + // + // TokenFile string `river:"token_file"` + + // Namespace is the name of the namespace to send along for the request + // when no other Namespace is present in the QueryOptions + Namespace string `river:"namespace,attr,optional"` + + // Allowlist of [Consul Metadata](https://www.consul.io/docs/agent/options#node_meta) + // keys to use as resource attributes. + MetaLabels []string `river:"meta,attr,optional"` + + // ResourceAttributes configuration for Consul detector + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + //TODO(ptodev): Change the OTel Collector's "meta" param to be a slice instead of a map. + var metaLabels map[string]string + if args.MetaLabels != nil { + metaLabels = make(map[string]string, len(args.MetaLabels)) + for _, label := range args.MetaLabels { + metaLabels[label] = "" + } + } + + return map[string]interface{}{ + "address": args.Address, + "datacenter": args.Datacenter, + "token": configopaque.String(args.Token), + "namespace": args.Namespace, + "meta": metaLabels, + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/consul resource attributes. +type ResourceAttributesConfig struct { + AzureResourcegroupName *rac.ResourceAttributeConfig `river:"azure.resourcegroup.name,block,optional"` + AzureVMName *rac.ResourceAttributeConfig `river:"azure.vm.name,block,optional"` + AzureVMScalesetName *rac.ResourceAttributeConfig `river:"azure.vm.scaleset.name,block,optional"` + AzureVMSize *rac.ResourceAttributeConfig `river:"azure.vm.size,block,optional"` + CloudAccountID *rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + HostID *rac.ResourceAttributeConfig `river:"host.id,block,optional"` + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "azure.resourcegroup.name": r.AzureResourcegroupName.Convert(), + "azure.vm.name": r.AzureVMName.Convert(), + "azure.vm.scaleset.name": r.AzureVMScalesetName.Convert(), + "azure.vm.size": r.AzureVMSize.Convert(), + "cloud.account.id": r.CloudAccountID.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "host.id": r.HostID.Convert(), + "host.name": r.HostName.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/docker/config.go b/component/otelcol/processor/resourcedetection/internal/docker/config.go new file mode 100644 index 000000000000..1dd7439ac2f9 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/docker/config.go @@ -0,0 +1,26 @@ +package docker + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/docker resource attributes. +type ResourceAttributesConfig struct { + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` + OsType *rac.ResourceAttributeConfig `river:"os.type,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "host.name": r.HostName.Convert(), + "os.type": r.OsType.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/gcp/config.go b/component/otelcol/processor/resourcedetection/internal/gcp/config.go new file mode 100644 index 000000000000..615f2f213cf6 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/gcp/config.go @@ -0,0 +1,56 @@ +package gcp + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/gcp resource attributes. +type ResourceAttributesConfig struct { + CloudAccountID *rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"` + CloudAvailabilityZone *rac.ResourceAttributeConfig `river:"cloud.availability_zone,block,optional"` + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + FaasID *rac.ResourceAttributeConfig `river:"faas.id,block,optional"` + FaasInstance *rac.ResourceAttributeConfig `river:"faas.instance,block,optional"` + FaasName *rac.ResourceAttributeConfig `river:"faas.name,block,optional"` + FaasVersion *rac.ResourceAttributeConfig `river:"faas.version,block,optional"` + GcpCloudRunJobExecution *rac.ResourceAttributeConfig `river:"gcp.cloud_run.job.execution,block,optional"` + GcpCloudRunJobTaskIndex *rac.ResourceAttributeConfig `river:"gcp.cloud_run.job.task_index,block,optional"` + GcpGceInstanceHostname *rac.ResourceAttributeConfig `river:"gcp.gce.instance.hostname,block,optional"` + GcpGceInstanceName *rac.ResourceAttributeConfig `river:"gcp.gce.instance.name,block,optional"` + HostID *rac.ResourceAttributeConfig `river:"host.id,block,optional"` + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` + HostType *rac.ResourceAttributeConfig `river:"host.type,block,optional"` + K8sClusterName *rac.ResourceAttributeConfig `river:"k8s.cluster.name,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.account.id": r.CloudAccountID.Convert(), + "cloud.availability_zone": r.CloudAvailabilityZone.Convert(), + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "faas.id": r.FaasID.Convert(), + "faas.instance": r.FaasInstance.Convert(), + "faas.name": r.FaasName.Convert(), + "faas.version": r.FaasVersion.Convert(), + "gcp.cloud_run.job.execution": r.GcpCloudRunJobExecution.Convert(), + "gcp.cloud_run.job.task_index": r.GcpCloudRunJobTaskIndex.Convert(), + "gcp.gce.instance.hostname": r.GcpGceInstanceHostname.Convert(), + "gcp.gce.instance.name": r.GcpGceInstanceName.Convert(), + "host.id": r.HostID.Convert(), + "host.name": r.HostName.Convert(), + "host.type": r.HostType.Convert(), + "k8s.cluster.name": r.K8sClusterName.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/heroku/config.go b/component/otelcol/processor/resourcedetection/internal/heroku/config.go new file mode 100644 index 000000000000..d1b1fbf81635 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/heroku/config.go @@ -0,0 +1,38 @@ +package heroku + +import rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" + +type Config struct { + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/heroku resource attributes. +type ResourceAttributesConfig struct { + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + HerokuAppID *rac.ResourceAttributeConfig `river:"heroku.app.id,block,optional"` + HerokuDynoID *rac.ResourceAttributeConfig `river:"heroku.dyno.id,block,optional"` + HerokuReleaseCommit *rac.ResourceAttributeConfig `river:"heroku.release.commit,block,optional"` + HerokuReleaseCreationTimestamp *rac.ResourceAttributeConfig `river:"heroku.release.creation_timestamp,block,optional"` + ServiceInstanceID *rac.ResourceAttributeConfig `river:"service.instance.id,block,optional"` + ServiceName *rac.ResourceAttributeConfig `river:"service.name,block,optional"` + ServiceVersion *rac.ResourceAttributeConfig `river:"service.version,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.provider": r.CloudProvider.Convert(), + "heroku.app.id": r.HerokuAppID.Convert(), + "heroku.dyno.id": r.HerokuDynoID.Convert(), + "heroku.release.commit": r.HerokuReleaseCommit.Convert(), + "heroku.release.creation_timestamp": r.HerokuReleaseCreationTimestamp.Convert(), + "service.instance.id": r.ServiceInstanceID.Convert(), + "service.name": r.ServiceName.Convert(), + "service.version": r.ServiceVersion.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/k8snode/config.go b/component/otelcol/processor/resourcedetection/internal/k8snode/config.go new file mode 100644 index 000000000000..ae39682f6a8e --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/k8snode/config.go @@ -0,0 +1,65 @@ +package k8snode + +import ( + "github.com/grafana/agent/component/otelcol" + rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" +) + +type Config struct { + KubernetesAPIConfig otelcol.KubernetesAPIConfig `river:",squash"` + // NodeFromEnv can be used to extract the node name from an environment + // variable. The value must be the name of the environment variable. + // This is useful when the node where an Agent will run on cannot be + // predicted. In such cases, the Kubernetes downward API can be used to + // add the node name to each pod as an environment variable. K8s tagger + // can then read this value and filter pods by it. + // + // For example, node name can be passed to each agent with the downward API as follows + // + // env: + // - name: K8S_NODE_NAME + // valueFrom: + // fieldRef: + // fieldPath: spec.nodeName + // + // Then the NodeFromEnv field can be set to `K8S_NODE_NAME` to filter all pods by the node that + // the agent is running on. + // + // More on downward API here: https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/ + NodeFromEnvVar string `river:"node_from_env_var,attr,optional"` + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +var DefaultConfig = Config{ + KubernetesAPIConfig: otelcol.KubernetesAPIConfig{ + AuthType: "none", + }, + NodeFromEnvVar: "K8S_NODE_NAME", +} + +func (c *Config) SetToDefault() { + *c = DefaultConfig +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + //TODO: K8sAPIConfig is squashed - is there a better way to "convert" it? + "auth_type": args.KubernetesAPIConfig.AuthType, + "context": args.KubernetesAPIConfig.Context, + "node_from_env_var": args.NodeFromEnvVar, + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/k8snode resource attributes. +type ResourceAttributesConfig struct { + K8sNodeName *rac.ResourceAttributeConfig `river:"k8s.node.name,block,optional"` + K8sNodeUID *rac.ResourceAttributeConfig `river:"k8s.node.uid,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "k8s.node.name": r.K8sNodeName.Convert(), + "k8s.node.uid": r.K8sNodeUID.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/openshift/config.go b/component/otelcol/processor/resourcedetection/internal/openshift/config.go new file mode 100644 index 000000000000..050dd77b4bbd --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/openshift/config.go @@ -0,0 +1,48 @@ +package openshift + +import ( + "github.com/grafana/agent/component/otelcol" + rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" +) + +// Config can contain user-specified inputs to overwrite default values. +// See `openshift.go#NewDetector` for more information. +type Config struct { + // Address is the address of the openshift api server + Address string `river:"address,attr,optional"` + + // Token is used to identify against the openshift api server + Token string `river:"token,attr,optional"` + + // TLSSettings contains TLS configurations that are specific to client + // connection used to communicate with the Openshift API. + TLSSettings otelcol.TLSClientArguments `river:"tls,block,optional"` + + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "address": args.Address, + "token": args.Token, + "tls": args.TLSSettings.Convert(), + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for openshift resource attributes. +type ResourceAttributesConfig struct { + CloudPlatform *rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"` + CloudProvider *rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"` + CloudRegion *rac.ResourceAttributeConfig `river:"cloud.region,block,optional"` + K8sClusterName *rac.ResourceAttributeConfig `river:"k8s.cluster.name,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "cloud.platform": r.CloudPlatform.Convert(), + "cloud.provider": r.CloudProvider.Convert(), + "cloud.region": r.CloudRegion.Convert(), + "k8s.cluster.name": r.K8sClusterName.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/resource_attribute_config/resource_attribute_config.go b/component/otelcol/processor/resourcedetection/internal/resource_attribute_config/resource_attribute_config.go new file mode 100644 index 000000000000..13b59b468738 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/resource_attribute_config/resource_attribute_config.go @@ -0,0 +1,28 @@ +package resource_attribute_config + +// Configures whether a resource attribute should be enabled or not. +type ResourceAttributeConfig struct { + // "enabled" as a mandatory parameter, because if this block is present in the config, + // it makes sense for the user to explicitly say whether they want to enable the attribute. + // + // Unlike the Collector, the Agent does not try to set a default value for "enabled" because: + // * Different resource attributes have different default values. + // It is time consuming to try to keep default values in sync with the Collector. + // * If we set a default value in the Agent, Collector will think that the user set it explicitly. + // This is due to an "enabledSetByUser" parameter which Collector uses to print warnings such as: + // * "[WARNING] Please set `enabled` field explicitly for `default.metric`: This metric will be disabled by default soon." + // * "[WARNING] `default.metric.to_be_removed` should not be enabled: This metric is deprecated and will be removed soon." + // Users who did not explicitly enable such resource attributes may be confused by these warnings. + // Therefore it's easier to not set a default value in the Agent. + Enabled bool `river:"enabled,attr"` +} + +func (r *ResourceAttributeConfig) Convert() map[string]interface{} { + if r == nil { + return nil + } + + return map[string]interface{}{ + "enabled": r.Enabled, + } +} diff --git a/component/otelcol/processor/resourcedetection/internal/system/config.go b/component/otelcol/processor/resourcedetection/internal/system/config.go new file mode 100644 index 000000000000..f46ca0256c44 --- /dev/null +++ b/component/otelcol/processor/resourcedetection/internal/system/config.go @@ -0,0 +1,76 @@ +package system + +import ( + "fmt" + + rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config" +) + +// Config defines user-specified configurations unique to the system detector +type Config struct { + // The HostnameSources is a priority list of sources from which hostname will be fetched. + // In case of the error in fetching hostname from source, + // the next source from the list will be considered. + HostnameSources []string `river:"hostname_sources,attr,optional"` + + ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block"` +} + +var DefaultConfig = Config{ + HostnameSources: []string{"dns", "os"}, +} + +func (c *Config) SetToDefault() { + *c = DefaultConfig +} + +// Validate config +func (cfg *Config) Validate() error { + for _, hostnameSource := range cfg.HostnameSources { + switch hostnameSource { + case "os", "dns", "cname", "lookup": + // Valid option - nothing to do + default: + return fmt.Errorf("invalid hostname source: %s", hostnameSource) + } + } + return nil +} + +func (args *Config) Convert() map[string]interface{} { + return map[string]interface{}{ + "hostname_sources": args.HostnameSources, + "resource_attributes": args.ResourceAttributes.Convert(), + } +} + +// ResourceAttributesConfig provides config for resourcedetectionprocessor/system resource attributes. +type ResourceAttributesConfig struct { + HostArch *rac.ResourceAttributeConfig `river:"host.arch,block,optional"` + HostCPUCacheL2Size *rac.ResourceAttributeConfig `river:"host.cpu.cache.l2.size,block,optional"` + HostCPUFamily *rac.ResourceAttributeConfig `river:"host.cpu.family,block,optional"` + HostCPUModelID *rac.ResourceAttributeConfig `river:"host.cpu.model.id,block,optional"` + HostCPUModelName *rac.ResourceAttributeConfig `river:"host.cpu.model.name,block,optional"` + HostCPUStepping *rac.ResourceAttributeConfig `river:"host.cpu.stepping,block,optional"` + HostCPUVendorID *rac.ResourceAttributeConfig `river:"host.cpu.vendor.id,block,optional"` + HostID *rac.ResourceAttributeConfig `river:"host.id,block,optional"` + HostName *rac.ResourceAttributeConfig `river:"host.name,block,optional"` + OsDescription *rac.ResourceAttributeConfig `river:"os.description,block,optional"` + OsType *rac.ResourceAttributeConfig `river:"os.type,block,optional"` +} + +func (r *ResourceAttributesConfig) Convert() map[string]interface{} { + return map[string]interface{}{ + "host.arch": r.HostArch.Convert(), + "host.cpu.cache.l2.size": r.HostCPUCacheL2Size.Convert(), + "host.cpu.family": r.HostCPUFamily.Convert(), + "host.cpu.model.id": r.HostCPUModelID.Convert(), + "host.cpu.model.name": r.HostCPUModelName.Convert(), + "host.cpu.stepping": r.HostCPUStepping.Convert(), + "host.cpu.vendor.id": r.HostCPUVendorID.Convert(), + "host.id": r.HostID.Convert(), + "host.name": r.HostName.Convert(), + "os.description": r.OsDescription.Convert(), + "os.type": r.OsType.Convert(), + } +} diff --git a/component/otelcol/processor/resourcedetection/resourcedetection.go b/component/otelcol/processor/resourcedetection/resourcedetection.go new file mode 100644 index 000000000000..f0cfebba3e5b --- /dev/null +++ b/component/otelcol/processor/resourcedetection/resourcedetection.go @@ -0,0 +1,204 @@ +package resourcedetection + +import ( + "fmt" + "time" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/otelcol" + "github.com/grafana/agent/component/otelcol/processor" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/aws/ec2" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/aws/ecs" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/aws/eks" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/aws/lambda" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/azure" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/azure/aks" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/consul" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/docker" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/gcp" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/heroku" + kubernetes_node "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/k8snode" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/openshift" + "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/system" + otel_service "github.com/grafana/agent/service/otel" + "github.com/grafana/river" + "github.com/mitchellh/mapstructure" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" + otelcomponent "go.opentelemetry.io/collector/component" + otelextension "go.opentelemetry.io/collector/extension" +) + +func init() { + component.Register(component.Registration{ + Name: "otelcol.processor.resourcedetection", + Args: Arguments{}, + Exports: otelcol.ConsumerExports{}, + NeedsServices: []string{otel_service.ServiceName}, + + Build: func(opts component.Options, args component.Arguments) (component.Component, error) { + fact := resourcedetectionprocessor.NewFactory() + return processor.New(opts, fact, args.(Arguments)) + }, + }) +} + +// Arguments configures the otelcol.processor.probabilistic_sampler component. +type Arguments struct { + // Detectors is an ordered list of named detectors that should be + // run to attempt to detect resource information. + Detectors []string `river:"detectors,attr"` + + // Override indicates whether any existing resource attributes + // should be overridden or preserved. Defaults to true. + Override bool `river:"override,attr,optional"` + + // DetectorConfig is a list of settings specific to all detectors + DetectorConfig DetectorConfig `river:",squash"` + + // HTTP client settings for the detector + // Timeout default is 5s + // Client otelcol.HTTPClientArguments `river:",squash"` + Timeout time.Duration `river:"timeout,attr,optional"` + //TODO: Uncomment this later? Can we just get away with a timeout, or do we need all the http client settings? + // It seems that HTTP client settings are only used in the ec2 detection via ClientFromContext. + + // Output configures where to send processed data. Required. + Output *otelcol.ConsumerArguments `river:"output,block"` +} + +// DetectorConfig contains user-specified configurations unique to all individual detectors +type DetectorConfig struct { + // EC2Config contains user-specified configurations for the EC2 detector + EC2Config ec2.Config `river:"ec2,block,optional"` + + // ECSConfig contains user-specified configurations for the ECS detector + ECSConfig ecs.Config `river:"ecs,block,optional"` + + // EKSConfig contains user-specified configurations for the EKS detector + EKSConfig eks.Config `river:"eks,block,optional"` + + // Elasticbeanstalk contains user-specified configurations for the elasticbeanstalk detector + ElasticbeanstalkConfig elasticbeanstalk.Config `river:"elasticbeanstalk,block,optional"` + + // Lambda contains user-specified configurations for the lambda detector + LambdaConfig lambda.Config `river:"lambda,block,optional"` + + // Azure contains user-specified configurations for the azure detector + AzureConfig azure.Config `river:"azure,block,optional"` + + // Aks contains user-specified configurations for the aks detector + AksConfig aks.Config `river:"aks,block,optional"` + + // ConsulConfig contains user-specified configurations for the Consul detector + ConsulConfig consul.Config `river:"consul,block,optional"` + + // DockerConfig contains user-specified configurations for the docker detector + DockerConfig docker.Config `river:"docker,block,optional"` + + // GcpConfig contains user-specified configurations for the gcp detector + GcpConfig gcp.Config `river:"gcp,block,optional"` + + // HerokuConfig contains user-specified configurations for the heroku detector + HerokuConfig heroku.Config `river:"heroku,block,optional"` + + // SystemConfig contains user-specified configurations for the System detector + SystemConfig system.Config `river:"system,block,optional"` + + // OpenShift contains user-specified configurations for the Openshift detector + OpenShiftConfig openshift.Config `river:"openshift,block,optional"` + + // KubernetesNode contains user-specified configurations for the K8SNode detector + KubernetesNodeConfig kubernetes_node.Config `river:"kubernetes_node,block,optional"` +} + +var ( + _ processor.Arguments = Arguments{} + _ river.Validator = (*Arguments)(nil) + _ river.Defaulter = (*Arguments)(nil) +) + +// DefaultArguments holds default settings for Arguments. +var DefaultArguments = Arguments{ + Override: true, + Timeout: 5 * time.Second, +} + +// SetToDefault implements river.Defaulter. +func (args *Arguments) SetToDefault() { + *args = DefaultArguments +} + +// Validate implements river.Validator. +func (args *Arguments) Validate() error { + if len(args.Detectors) == 0 { + return fmt.Errorf("at least one detector must be specified") + } + return nil +} + +func (args Arguments) ConvertDetectors() []string { + if args.Detectors == nil { + return nil + } + + res := make([]string, 0, len(args.Detectors)) + for _, detector := range args.Detectors { + //TODO(ptodev): Check if the detector name is valid + switch detector { + case "kubernetes_node": + res = append(res, "k8snode") + default: + res = append(res, detector) + } + } + return res +} + +// Convert implements processor.Arguments. +func (args Arguments) Convert() (otelcomponent.Config, error) { + input := make(map[string]interface{}) + + input["detectors"] = args.ConvertDetectors() + input["override"] = args.Override + input["timeout"] = args.Timeout + + input["ec2"] = args.DetectorConfig.EC2Config.Convert() + input["ecs"] = args.DetectorConfig.ECSConfig.Convert() + input["eks"] = args.DetectorConfig.EKSConfig.Convert() + input["elasticbeanstalk"] = args.DetectorConfig.ElasticbeanstalkConfig.Convert() + input["lambda"] = args.DetectorConfig.LambdaConfig.Convert() + input["azure"] = args.DetectorConfig.AzureConfig.Convert() + input["aks"] = args.DetectorConfig.AksConfig.Convert() + input["consul"] = args.DetectorConfig.ConsulConfig.Convert() + input["docker"] = args.DetectorConfig.DockerConfig.Convert() + input["gcp"] = args.DetectorConfig.GcpConfig.Convert() + input["heroku"] = args.DetectorConfig.HerokuConfig.Convert() + input["system"] = args.DetectorConfig.SystemConfig.Convert() + input["openshift"] = args.DetectorConfig.OpenShiftConfig.Convert() + input["k8snode"] = args.DetectorConfig.KubernetesNodeConfig.Convert() + + var result resourcedetectionprocessor.Config + err := mapstructure.Decode(input, &result) + + if err != nil { + return nil, err + } + + return &result, nil +} + +// Extensions implements processor.Arguments. +func (args Arguments) Extensions() map[otelcomponent.ID]otelextension.Extension { + return nil +} + +// Exporters implements processor.Arguments. +func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.ID]otelcomponent.Component { + return nil +} + +// NextConsumers implements processor.Arguments. +func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { + return args.Output +} diff --git a/component/otelcol/processor/resourcedetection/resourcedetection_test.go b/component/otelcol/processor/resourcedetection/resourcedetection_test.go new file mode 100644 index 000000000000..16f8f23f94db --- /dev/null +++ b/component/otelcol/processor/resourcedetection/resourcedetection_test.go @@ -0,0 +1,922 @@ +package resourcedetection_test + +import ( + "testing" + "time" + + "github.com/grafana/agent/component/otelcol/processor/resourcedetection" + "github.com/grafana/river" + "github.com/mitchellh/mapstructure" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" + "github.com/stretchr/testify/require" +) + +func TestArguments_UnmarshalRiver(t *testing.T) { + tests := []struct { + testName string + cfg string + expected map[string]interface{} + errorMsg string + }{ + { + testName: "err_no_detector", + cfg: ` + output {} + `, + errorMsg: "at least one detector must be specified", + }, + { + testName: "ec2_defaults", + cfg: ` + detectors = ["ec2"] + ec2 { + resource_attributes { + cloud.account.id { enabled = true } + cloud.availability_zone { enabled = true } + cloud.platform { enabled = true } + cloud.provider { enabled = true } + cloud.region { enabled = true } + host.id { enabled = true } + host.image.id { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"ec2"}, + "timeout": 5 * time.Second, + "override": true, + "ec2": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.account.id": map[string]interface{}{ + "enabled": true, + }, + "cloud.availability_zone": map[string]interface{}{ + "enabled": true, + }, + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "cloud.region": map[string]interface{}{ + "enabled": true, + }, + "host.id": map[string]interface{}{ + "enabled": true, + }, + "host.image.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + "host.type": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "ec2_explicit", + cfg: ` + detectors = ["ec2"] + ec2 { + tags = ["^tag1$", "^tag2$", "^label.*$"] + resource_attributes { + cloud.account.id { enabled = true } + cloud.availability_zone { enabled = true } + cloud.platform { enabled = true } + cloud.provider { enabled = true } + cloud.region { enabled = true } + host.id { enabled = true } + host.image.id { enabled = false } + host.name { enabled = false } + host.type { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"ec2"}, + "timeout": 5 * time.Second, + "override": true, + "ec2": map[string]interface{}{ + "tags": []string{"^tag1$", "^tag2$", "^label.*$"}, + "resource_attributes": map[string]interface{}{ + "cloud.account.id": map[string]interface{}{ + "enabled": true, + }, + "cloud.availability_zone": map[string]interface{}{ + "enabled": true, + }, + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "cloud.region": map[string]interface{}{ + "enabled": true, + }, + "host.id": map[string]interface{}{ + "enabled": true, + }, + "host.image.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + "host.type": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "ecs", + cfg: ` + detectors = ["ecs"] + ecs { + resource_attributes { + aws.ecs.cluster.arn { enabled = true } + aws.ecs.launchtype { enabled = true } + aws.ecs.task.arn { enabled = true } + aws.ecs.task.family { enabled = true } + aws.ecs.task.revision { enabled = true } + aws.log.group.arns { enabled = true } + aws.log.group.names { enabled = false } + // aws.log.stream.arns { enabled = true } + // aws.log.stream.names { enabled = true } + // cloud.account.id { enabled = true } + // cloud.availability_zone { enabled = true } + // cloud.platform { enabled = true } + // cloud.provider { enabled = true } + // cloud.region { enabled = true } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"ecs"}, + "timeout": 5 * time.Second, + "override": true, + "ecs": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "aws.ecs.cluster.arn": map[string]interface{}{ + "enabled": true, + }, + "aws.ecs.launchtype": map[string]interface{}{ + "enabled": true, + }, + "aws.ecs.task.arn": map[string]interface{}{ + "enabled": true, + }, + "aws.ecs.task.family": map[string]interface{}{ + "enabled": true, + }, + "aws.ecs.task.revision": map[string]interface{}{ + "enabled": true, + }, + "aws.log.group.arns": map[string]interface{}{ + "enabled": true, + }, + "aws.log.group.names": map[string]interface{}{ + "enabled": false, + }, + "aws.log.stream.arns": map[string]interface{}{ + "enabled": false, + }, + "aws.log.stream.names": map[string]interface{}{ + "enabled": false, + }, + "cloud.account.id": map[string]interface{}{ + "enabled": false, + }, + "cloud.availability_zone": map[string]interface{}{ + "enabled": false, + }, + "cloud.platform": map[string]interface{}{ + "enabled": false, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "eks", + cfg: ` + detectors = ["eks"] + eks { + resource_attributes { + cloud.platform { enabled = true } + cloud.provider { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"eks"}, + "timeout": 5 * time.Second, + "override": true, + "eks": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "azure", + cfg: ` + detectors = ["azure"] + azure { + resource_attributes { + azure.resourcegroup.name { enabled = true } + azure.vm.name { enabled = true } + azure.vm.scaleset.name { enabled = true } + azure.vm.size { enabled = true } + cloud.account.id { enabled = true } + cloud.platform { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"azure"}, + "timeout": 5 * time.Second, + "override": true, + "azure": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "azure.resourcegroup.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.scaleset.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.size": map[string]interface{}{ + "enabled": true, + }, + "cloud.account.id": map[string]interface{}{ + "enabled": true, + }, + "cloud.platform": map[string]interface{}{ + "enabled": false, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "host.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "aks", + cfg: ` + detectors = ["aks"] + aks { + resource_attributes { + cloud.platform { enabled = true } + cloud.provider { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"aks"}, + "timeout": 5 * time.Second, + "override": true, + "aks": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "gcp", + cfg: ` + detectors = ["gcp"] + gcp { + resource_attributes { + cloud.account.id { enabled = true } + cloud.availability_zone { enabled = true } + cloud.platform { enabled = true } + cloud.provider { enabled = true } + cloud.region { enabled = false } + faas.id { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"gcp"}, + "timeout": 5 * time.Second, + "override": true, + "gcp": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.account.id": map[string]interface{}{ + "enabled": true, + }, + "cloud.availability_zone": map[string]interface{}{ + "enabled": true, + }, + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "faas.id": map[string]interface{}{ + "enabled": false, + }, + "faas.instance": map[string]interface{}{ + "enabled": false, + }, + "faas.name": map[string]interface{}{ + "enabled": false, + }, + "faas.version": map[string]interface{}{ + "enabled": false, + }, + "gcp.cloud_run.job.execution": map[string]interface{}{ + "enabled": false, + }, + "gcp.cloud_run.job.task_index": map[string]interface{}{ + "enabled": false, + }, + "gcp.gce.instance.hostname": map[string]interface{}{ + "enabled": false, + }, + "gcp.gce.instance.name": map[string]interface{}{ + "enabled": false, + }, + "host.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + "host.type": map[string]interface{}{ + "enabled": false, + }, + "k8s.cluster.name": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "docker", + cfg: ` + detectors = ["docker"] + docker { + resource_attributes { + host.name { enabled = true } + os.type { enabled = false } + + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"docker"}, + "timeout": 5 * time.Second, + "override": true, + "docker": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "host.name": map[string]interface{}{ + "enabled": true, + }, + "os.type": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "lambda", + cfg: ` + detectors = ["lambda"] + lambda { + resource_attributes { + aws.log.group.names { enabled = true } + aws.log.stream.names { enabled = true } + cloud.platform { enabled = true } + cloud.provider { enabled = false } + cloud.region { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"lambda"}, + "timeout": 5 * time.Second, + "override": true, + "lambda": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "aws.log.group.names": map[string]interface{}{ + "enabled": true, + }, + "aws.log.stream.names": map[string]interface{}{ + "enabled": true, + }, + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "faas.instance": map[string]interface{}{ + "enabled": false, + }, + "faas.max_memory": map[string]interface{}{ + "enabled": false, + }, + "faas.name": map[string]interface{}{ + "enabled": false, + }, + "faas.version": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "elasticbeanstalk", + cfg: ` + detectors = ["elasticbeanstalk"] + elasticbeanstalk { + resource_attributes { + cloud.platform { enabled = true } + cloud.provider { enabled = true } + deployment.environment { enabled = true } + service.instance.id { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"elasticbeanstalk"}, + "timeout": 5 * time.Second, + "override": true, + "elasticbeanstalk": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "deployment.environment": map[string]interface{}{ + "enabled": true, + }, + "service.instance.id": map[string]interface{}{ + "enabled": false, + }, + "service.version": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "consul_defaults", + cfg: ` + detectors = ["consul"] + consul { + resource_attributes { } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"consul"}, + "timeout": 5 * time.Second, + "override": true, + "consul": map[string]interface{}{ + "address": "", + "datacenter": "", + "token": "", + "namespace": "", + "meta": nil, + "resource_attributes": map[string]interface{}{ + "azure.resourcegroup.name": map[string]interface{}{ + "enabled": false, + }, + "azure.vm.name": map[string]interface{}{ + "enabled": false, + }, + "azure.vm.scaleset.name": map[string]interface{}{ + "enabled": false, + }, + "azure.vm.size": map[string]interface{}{ + "enabled": false, + }, + "cloud.account.id": map[string]interface{}{ + "enabled": false, + }, + "cloud.platform": map[string]interface{}{ + "enabled": false, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "host.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "consul_explicit", + cfg: ` + detectors = ["consul"] + consul { + address = "localhost:8500" + datacenter = "dc1" + token = "secret_token" + namespace = "test_namespace" + meta = ["test"] + resource_attributes { + azure.resourcegroup.name { enabled = true } + azure.vm.name { enabled = true } + azure.vm.scaleset.name { enabled = true } + azure.vm.size { enabled = true } + cloud.account.id { enabled = false } + cloud.platform { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"consul"}, + "timeout": 5 * time.Second, + "override": true, + "consul": map[string]interface{}{ + "address": "localhost:8500", + "datacenter": "dc1", + "token": "secret_token", + "namespace": "test_namespace", + "meta": map[string]string{"test": ""}, + "resource_attributes": map[string]interface{}{ + "azure.resourcegroup.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.scaleset.name": map[string]interface{}{ + "enabled": true, + }, + "azure.vm.size": map[string]interface{}{ + "enabled": true, + }, + "cloud.account.id": map[string]interface{}{ + "enabled": false, + }, + "cloud.platform": map[string]interface{}{ + "enabled": false, + }, + "cloud.provider": map[string]interface{}{ + "enabled": false, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "host.id": map[string]interface{}{ + "enabled": false, + }, + "host.name": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "heroku", + cfg: ` + detectors = ["heroku"] + heroku { + resource_attributes { + cloud.provider { enabled = true } + heroku.app.id { enabled = true } + heroku.dyno.id { enabled = true } + heroku.release.commit { enabled = true } + heroku.release.creation_timestamp { enabled = false } + service.instance.id { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"heroku"}, + "timeout": 5 * time.Second, + "override": true, + "heroku": map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "heroku.app.id": map[string]interface{}{ + "enabled": true, + }, + "heroku.dyno.id": map[string]interface{}{ + "enabled": true, + }, + "heroku.release.commit": map[string]interface{}{ + "enabled": true, + }, + "heroku.release.creation_timestamp": map[string]interface{}{ + "enabled": false, + }, + "service.instance.id": map[string]interface{}{ + "enabled": false, + }, + "service.name": map[string]interface{}{ + "enabled": false, + }, + "service.version": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "kubernetes_node_defaults", + cfg: ` + detectors = ["kubernetes_node"] + kubernetes_node { + resource_attributes { } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"k8snode"}, + "timeout": 5 * time.Second, + "override": true, + "k8snode": map[string]interface{}{ + "auth_type": "none", + "node_from_env_var": "K8S_NODE_NAME", + "resource_attributes": map[string]interface{}{ + "k8s.node.name": map[string]interface{}{ + "enabled": false, + }, + "k8s.node.uid": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "kubernetes_node_explicit", + cfg: ` + detectors = ["kubernetes_node"] + kubernetes_node { + auth_type = "kubeConfig" + context = "fake_ctx" + node_from_env_var = "MY_CUSTOM_VAR" + resource_attributes { + k8s.node.name { enabled = true } + k8s.node.uid { enabled = false } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"k8snode"}, + "timeout": 5 * time.Second, + "override": true, + "k8snode": map[string]interface{}{ + "auth_type": "kubeConfig", + "context": "fake_ctx", + "node_from_env_var": "MY_CUSTOM_VAR", + "resource_attributes": map[string]interface{}{ + "k8s.node.name": map[string]interface{}{ + "enabled": true, + }, + "k8s.node.uid": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + { + testName: "system_explicit", + cfg: ` + detectors = ["system"] + system { + hostname_sources = ["os"] + resource_attributes { + host.arch { enabled = true } + host.cpu.cache.l2.size { enabled = true } + host.cpu.family { enabled = true } + host.cpu.model.id { enabled = true } + host.cpu.model.name { enabled = true } + host.cpu.stepping { enabled = true } + host.cpu.vendor.id { enabled = false } + host.id { enabled = false } + host.name { enabled = false } + // os.description { enabled = true } + // os.type { enabled = true } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"system"}, + "timeout": 5 * time.Second, + "override": true, + "system": map[string]interface{}{ + "hostname_sources": []string{"os"}, + "resource_attributes": map[string]interface{}{ + "host.arch": map[string]interface{}{ + "enabled": true, + }, + "host.cpu.cache.l2.size": map[string]interface{}{ + "enabled": true, + }, + "host.cpu.family": map[string]interface{}{ + "enabled": true, + }, + "host.cpu.model.id": map[string]interface{}{ + "enabled": true, + }, + "host.cpu.model.name": map[string]interface{}{ + "enabled": true, + }, + "host.cpu.stepping": map[string]interface{}{ + "enabled": true, + }, + }, + }, + }, + }, + { + testName: "system_defaults", + cfg: ` + detectors = ["system"] + system { + hostname_sources = ["asdf"] + resource_attributes { } + } + output {} + `, + errorMsg: "invalid hostname source: asdf", + }, + { + testName: "openshift_default", + cfg: ` + detectors = ["openshift"] + openshift { + resource_attributes {} + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"openshift"}, + "timeout": 5 * time.Second, + "override": true, + "openshift": map[string]interface{}{ + "address": "", + "token": "", + // "tls": map[string]interface{}{ + // "insecure": true, + // }, + }, + }, + }, + { + testName: "openshift", + cfg: ` + detectors = ["openshift"] + timeout = "7s" + override = false + openshift { + address = "127.0.0.1:4444" + token = "some_token" + tls { + insecure = true + } + resource_attributes { + cloud.platform { + enabled = true + } + cloud.provider { + enabled = true + } + cloud.region { + enabled = false + } + k8s.cluster.name { + enabled = false + } + } + } + output {} + `, + expected: map[string]interface{}{ + "detectors": []string{"openshift"}, + "timeout": 7 * time.Second, + "override": false, + "openshift": map[string]interface{}{ + "address": "127.0.0.1:4444", + "token": "some_token", + "tls": map[string]interface{}{ + "insecure": true, + }, + "resource_attributes": map[string]interface{}{ + "cloud.platform": map[string]interface{}{ + "enabled": true, + }, + "cloud.provider": map[string]interface{}{ + "enabled": true, + }, + "cloud.region": map[string]interface{}{ + "enabled": false, + }, + "k8s.cluster.name": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.testName, func(t *testing.T) { + var args resourcedetection.Arguments + err := river.Unmarshal([]byte(tc.cfg), &args) + if tc.errorMsg != "" { + require.ErrorContains(t, err, tc.errorMsg) + return + } + + require.NoError(t, err) + + actualPtr, err := args.Convert() + require.NoError(t, err) + + actual := actualPtr.(*resourcedetectionprocessor.Config) + + var expected resourcedetectionprocessor.Config + err = mapstructure.Decode(tc.expected, &expected) + require.NoError(t, err) + + require.Equal(t, expected, *actual) + }) + } +} diff --git a/docs/sources/flow/reference/components/otelcol.processor.resourcedetection.md b/docs/sources/flow/reference/components/otelcol.processor.resourcedetection.md new file mode 100644 index 000000000000..6e4e56b4511d --- /dev/null +++ b/docs/sources/flow/reference/components/otelcol.processor.resourcedetection.md @@ -0,0 +1,261 @@ +--- +canonical: https://grafana.com/docs/agent/latest/flow/reference/components/otelcol.processor.resourcedetection/ +labels: + stage: beta +title: otelcol.processor.resourcedetection +description: Learn about otelcol.processor.resourcedetection +--- + +# otelcol.processor.resourcedetection + +{{< docs/shared lookup="flow/stability/beta.md" source="agent" version="" >}} + +`otelcol.processor.resourcedetection` detects resource information and adds resource attributes to logs, +metrics, and traces telemetry data. + +{{% admonition type="note" %}} +`otelcol.processor.resourcedetection` is a wrapper over the upstream +OpenTelemetry Collector Contrib `resourcedetection` processor. If necessary, +bug reports or feature requests will be redirected to the upstream repository. +{{% /admonition %}} + +You can specify multiple `otelcol.processor.resourcedetection` components by giving them +different labels. + +## Usage + +```river +otelcol.processor.resourcedetection "LABEL" { + output { + logs = [...] + traces = [...] + } +} +``` + +## Arguments + +`otelcol.processor.resourcedetection` supports the following arguments: + +Name | Type | Description | Default | Required +----------- | -------------- | -------------------------------------------------------------------------------------- |-------- | -------- +`detectors` | `list(string)` | An ordered list of named detectors which should be ran to detect resource information. | | yes +`override` | `bool` | Configures whether existing resource attributes should be overriden or preserved. | `true` | no +`timeout` | `duration` | Timeout by which all specified detectors must complete. | `"5s"` | no + +## Blocks + +The following blocks are supported inside the definition of +`otelcol.processor.resourcedetection`: + +Hierarchy | Block | Description | Required +-------------------------------------- | ---------------------------------------- | ------------------------------------------------- | -------- +output | [output][] | Configures where to send received telemetry data. | yes +ec2 | [ec2][] | Configures where to send received telemetry data. | no +ec2 > resource_attributes | [ec2-resource_attributes][] | Configures where to send received telemetry data. | no +ecs | [ecs][] | Configures where to send received telemetry data. | no +ecs > resource_attributes | [ecs-resource_attributes][] | Configures where to send received telemetry data. | no +eks | [eks][] | Configures where to send received telemetry data. | no +eks > resource_attributes | [eks-resource_attributes][] | Configures where to send received telemetry data. | no +elasticbeanstalk | [elasticbeanstalk][] | Configures where to send received telemetry data. | no +elasticbeanstalk > resource_attributes | [elasticbeanstalk-resource_attributes][] | Configures where to send received telemetry data. | no +lambda | [lambda][] | Configures where to send received telemetry data. | no +lambda > resource_attributes | [lambda-resource_attributes][] | Configures where to send received telemetry data. | no +azure | [azure][] | Configures where to send received telemetry data. | no +azure > resource_attributes | [azure-resource_attributes][] | Configures where to send received telemetry data. | no +aks | [aks][] | Configures where to send received telemetry data. | no +aks > resource_attributes | [aks-resource_attributes][] | Configures where to send received telemetry data. | no +consul | [consul][] | Configures where to send received telemetry data. | no +consul > resource_attributes | [consul-resource_attributes][] | Configures where to send received telemetry data. | no +docker | [docker][] | Configures where to send received telemetry data. | no +docker > resource_attributes | [docker-resource_attributes][] | Configures where to send received telemetry data. | no +gcp | [gcp][] | Configures where to send received telemetry data. | no +gcp > resource_attributes | [gcp-resource_attributes][] | Configures where to send received telemetry data. | no +heroku | [heroku][] | Configures where to send received telemetry data. | no +heroku > resource_attributes | [heroku-resource_attributes][] | Configures where to send received telemetry data. | no +system | [system][] | Configures where to send received telemetry data. | no +system > resource_attributes | [system-resource_attributes][] | Configures where to send received telemetry data. | no +openshift | [openshift][] | Configures where to send received telemetry data. | no +openshift > resource_attributes | [openshift-resource_attributes][] | Configures where to send received telemetry data. | no +kubernetes_node | [kubernetes_node][] | Configures where to send received telemetry data. | no +kubernetes_node > resource_attributes | [kubernetes_node-resource_attributes][] | Configures where to send received telemetry data. | no + +[output]: #output-block +[ec2]: #ec2-block +[ec2-resource_attributes]: #ec2-resource_attributes-block +[ecs]: #ecs-block +[ecs-resource_attributes]: #ecs-resource_attributes-block +[eks]: #eks-block +[eks-resource_attributes]: #eks-resource_attributes-block +[elasticbeanstalk]: #elasticbeanstalk-block +[elasticbeanstalk-resource_attributes]: #elasticbeanstalk-resource_attributes-block +[lambda]: #lambda-block +[lambda-resource_attributes]: #lambda-resource_attributes-block +[azure]: #azure-block +[azure-resource_attributes]: #azure--resource_attributesblock +[aks]: #aks-block +[aks-resource_attributes]: #aks-resource_attributes-block +[consul]: #consul-block +[consul-resource_attributes]: #consul-resource_attributes-block +[docker]: #docker-block +[docker-resource_attributes]: #docker-resource_attributes-block +[gcp]: #gcp-block +[gcp-resource_attributes]: #gcp-resource_attributes-block +[heroku]: #heroku-block +[heroku-resource_attributes]: #heroku-resource_attributes-block +[system]: #system-block +[system-resource_attributes]: #system-resource_attributes-block +[openshift]: #openshift-block +[openshift-resource_attributes]: #openshift-resource_attributes-block +[kubernetes_node]: #kubernetes_node-block +[kubernetes_node-resource_attributes]: #kubernetes_node-resource_attributes-block + +### output block + +{{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" version="" >}} + +### ec2 block + +The `ec2` block uses [AWS SDK for Go](https://docs.aws.amazon.com/sdk-for-go/api/aws/ec2metadata/) to read +resource information from the [EC2 instance metadata API](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html). + +The following attributes are supported: + +Name | Type | Description | Default | Required +------ |----------------|--------------------------------------|-------------| -------- +`tags` | `list(string)` | | `[]` | no + +### ec2 > resource_attributes block + +The following attributes are supported: + +Name | Type | Description | Default | Required +------ |----------------|--------------------------------------|-------------| -------- +`name` | `string` | | `[]` | no + +### ecs block + +### ecs > resource_attributes block + +### eks block + +### eks > resource_attributes block + +### elasticbeanstalk block + +### elasticbeanstalk > resource_attributes block + +### lambda block + +### lambda > resource_attributes block + +### azure block + +### azure > resource_attributes block + +### aks block + +### aks > resource_attributes block + +### consul block + +### consul > resource_attributes block + +### docker block + +### docker > resource_attributes block + +### gcp block + +### gcp > resource_attributes block + +### heroku block + +### heroku > resource_attributes block + +### system block + +### system > resource_attributes block + +### openshift block + +### openshift > resource_attributes block + +### kubernetes_node block + +### kubernetes_node > resource_attributes block + +## Exported fields + +The following fields are exported and can be referenced by other components: + +Name | Type | Description +---- | ---- | ----------- +`input` | `otelcol.Consumer` | A value that other components can use to send telemetry data to. + +`input` accepts `otelcol.Consumer` OTLP-formatted data for any telemetry signal of these types: +* logs +* metrics +* traces + +## Component health + +`otelcol.processor.resourcedetection` is only reported as unhealthy if given an invalid +configuration. + +## Debug information + +`otelcol.processor.resourcedetection` does not expose any component-specific debug +information. + +## Examples + +### Basic usage + +```river +otelcol.processor.resourcedetection "default" { + + output { + logs = [otelcol.exporter.otlp.default.input] + } +} +``` + +### Sample 15% of the logs + +```river +otelcol.processor.resourcedetection "default" { + sampling_percentage = 15 + + output { + logs = [otelcol.exporter.otlp.default.input] + } +} +``` + +### Sample logs according to their "logID" attribute + +```river +otelcol.processor.resourcedetection "default" { + sampling_percentage = 15 + attribute_source = "record" + from_attribute = "logID" + + output { + logs = [otelcol.exporter.otlp.default.input] + } +} +``` + +### Sample logs according to a "priority" attribute + +```river +otelcol.processor.resourcedetection "default" { + sampling_percentage = 15 + sampling_priority = "priority" + + output { + logs = [otelcol.exporter.otlp.default.input] + } +} +``` diff --git a/go.mod b/go.mod index 3f9215d560fe..733da560ba75 100644 --- a/go.mod +++ b/go.mod @@ -117,6 +117,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.87.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/servicegraphprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanprocessor v0.87.0 @@ -622,7 +623,9 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.20.0 // indirect github.com/Shopify/sarama v1.38.1 // indirect + github.com/Showmax/go-fqdn v1.0.0 // indirect github.com/Workiva/go-datastructures v1.1.0 // indirect github.com/drone/envsubst v1.0.3 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect @@ -633,8 +636,11 @@ require ( github.com/leoluk/perflib_exporter v0.2.0 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.87.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.87.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.87.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.87.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.87.0 // indirect github.com/openshift/api v3.9.0+incompatible // indirect github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 // indirect github.com/prometheus-community/prom-label-proxy v0.6.0 // indirect diff --git a/go.sum b/go.sum index ae7048887316..bf2023b99a75 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/DataDog/datadog-go v0.0.0-20160329135253-cc2f4770f4d6/go.mod h1:LButx github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.20.0 h1:tk85AYGwOf6VNtoOQi8w/kVDi2vmPxp3/OU2FsUpdcA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.20.0/go.mod h1:Xx0VKh7GJ4si3rmElbh19Mejxz68ibWg/J30ZOMrqzU= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/IBM/sarama v1.41.2 h1:ZDBZfGPHAD4uuAtSv4U22fRZBgst0eEwGFzLj0fb85c= @@ -239,6 +241,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWso github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= +github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= +github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -1751,6 +1755,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2client github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension v0.87.0/go.mod h1:DRpgdIDMa+CFE96SoEPwigGBuZbwSNWotTgkJlrZMVc= github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension v0.87.0 h1:Z4o71/rS7mmpJ/9uzta3/nTaT+vKt0CU35o4inDLA9Y= github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension v0.87.0/go.mod h1:clScLUe8m0CTZMcV0scqq+fFFvw5Q1dASkYlYsrRptM= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.87.0 h1:JJsQ6iMFIDb7W6uLh6LQ5k4XOgWolr7ugVBoeV4l7hQ= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.87.0/go.mod h1:rDdtaUrMV6TJHqssyiYSfsLfFN1pIg4JOTDaE9AUapQ= github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.87.0 h1:W4Ty2pSyge/qNAOILO6HqyKrAcgALs0bn5CmpGZJXVo= github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.87.0/go.mod h1:3EFmVoLcdM8Adj75N8TGJ4txDB29oW1chTLCFiL/wxs= github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.87.0 h1:ekT4/I9J484j4yR/0VHj5AGtgv8KmNd+e4oXxNJNR/o= @@ -1763,6 +1769,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8stest v0.87 github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8stest v0.87.0/go.mod h1:ntSfqIeoGj0O+pXXyqDG9iTAw/PQg2JsO26EJ1GAKto= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.87.0 h1:kDamu7uZHRmeJWqaJg42LSgprRGokmQ4t8ACslzS0GU= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.87.0/go.mod h1:EAw9aBkrDIDWQvRBdJiDkaJmCqcgZpiZzYZEvOjg4uI= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.87.0 h1:8pVElJ4AMIiJxS+sxnK9CX73RED7iv/FYbqkvvX01ig= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.87.0/go.mod h1:zRQU4eN6rNXeVKD8g2p2Czb88o/Hd2BkVdar5nCk0+k= github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.87.0 h1:sx1ye7Y2rJ2qi11i2ih9T7BocxaV0uaBBf7B8ijCYpU= github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.87.0/go.mod h1:AobBiNPFNHUm0MJFTieajasG/xNMjMYI7BGGTSKh0xg= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.87.0 h1:sy75u6ZwBvRwv9RjEF65SqlkBsAeZFqF4+eFOLhIsJQ= @@ -1793,6 +1801,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattribute github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.87.0/go.mod h1:g6H0fB9TW03Lb8M+H0BXtgQp7gPncIwf3Fk73xOs9EA= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.87.0 h1:QJKdtNcsxBhG2ZwSzYRVI0oxUqBJJvhfWf0OnjHU3jY= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.87.0/go.mod h1:skMmFcl+gxyiOQXvwHc0IKpC73iyQ7zl9r1aRNmPMwI= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.87.0 h1:gEv7UNu4K5ptvKIpWQmVS+0XMrIzqZWczcjyhLnsx9M= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.87.0/go.mod h1:6Rnjwj4bZU7Ab+nLD1YqQlbdsnsKoOR/OzyI42+PyE8= github.com/open-telemetry/opentelemetry-collector-contrib/processor/servicegraphprocessor v0.87.0 h1:BIGb6dfmaTlDE7KbiQUhnD9SvL5HanbJbWJrnzURfPY= github.com/open-telemetry/opentelemetry-collector-contrib/processor/servicegraphprocessor v0.87.0/go.mod h1:EnaQxXfCCWkSEfsQbGOvYbeJ/EuqvtMYTLTq8RN6TiY= github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor v0.87.0 h1:4l/QetnprIMethZYfD2RK+MfMR83f6QycYb9bhJFItc=