v2: Converting Loki logs to OTLP.

[cxk314]

Hi,
I am trying to scrape metrics, logs and traces and send everything to a single Grafana Cloud OTLP endpoint, but it appears that conversion from Loki logs to OTLP drops all index labels. These are not OTLP logs coming from OTLP instrumented applications but regular node logs and pod logs that I am sending to OTLP endpoint. There is some conversion from Loki format to OTLP. Does k8s-monitoring-helm (Alloy) support conversion between Loki logs to OTLP? Or I need to send Loki logs to Loki endpoint and OTLP logs from instrumented applications to OTLP endpoint? Here is the config I am trying:

cluster:
  name: gcp-sb-cluster

destinations:
  - name: otlp-grafana-cloud
    type: otlp
    url: https://otlp-gateway-prod-us-east-0.grafana.net/otlp
    protocol: http
    auth:
      type: basic
      username: <username>
      password:  <password>
    processors:
      batch:
        size: 2000
        maxSize: 2000
      k8sattributes:
        # -- Kubernetes metadata to extract and add to the attributes of the received telemetry data.
        # @section -- Processors: K8s Attributes
        metadata:
          - container.name
          - deployment.environment
          - k8s.cluster.name
          - k8s.container.name
          - k8s.replicaset.name
          - k8s.namespace.name
          - k8s.pod.name
          - k8s.deployment.name
          - service.name
          - service.namespace
          - k8s.statefulset.name
          - service.instance.id
          - k8s.daemonset.name
          - k8s.cronjob.name
          - k8s.job.name
          - k8s.node.name
          - k8s.pod.uid
          - k8s.pod.start_time
      attributes:
        actions:
          - key: TENANT_ID
            action: upsert
            value: gcp-sb-cluster

      transform:
        metrics:
          resource:
            - set(attributes["CUSTOM_TENANT_ID"], "Test_Custom_Label") where attributes["k8s.namespace.name"] == "dapr-system"
            - set(attributes["TENANT_ID"], "gcp-sb-cluster-resource") 
        logs:
            log:
              - set(resource.attributes["TENANT_ID"], "gcp-sb-cluster")
        traces:
          resource:
            - set(attributes["CUSTOM_TENANT_ID"], "Test_Custom_Label") where attributes["k8s.namespace.name"] == "dapr-system"
            - set(attributes["TENANT_ID"], "gcp-sb-cluster-resource")
    metrics:
      # -- Whether to send metrics to the OTLP destination.
      # @section -- Telemetry
      enabled: true
    logs:
      # -- Whether to send logs to the OTLP destination.
      # @section -- Telemetry
      enabled: true
    tls:
      # -- Disables validation of the server certificate.
      # @section -- TLS
      insecureSkipVerify: true

clusterMetrics:
  enabled: true
  kube-state-metrics:
    podAnnotations: {kubernetes.azure.com/set-kube-service-host-fqdn: "true"}
  destinations: [otlp, otlp-grafana-cloud]

clusterEvents:
  enabled: true
  destinations: [otlp, otlp-grafana-cloud]

podLogs:
  enabled: true
  destinations: [otlp, otlp-grafana-cloud]

nodeLogs:
  enabled: true
  destinations: [otlp, otlp-grafana-cloud]

windows-exporter:
  deploy: false

selfReporting:
  enabled: false

# -- Application Observability.
# Requires destinations that supports metrics, logs, and traces.
# To see the valid options, please see the [Application Observability feature documentation](https://github.com/grafana/k8s-monitoring-helm/tree/main/charts/feature-application-observability).
# @default -- Disabled
# @section -- Features - Application Observability
applicationObservability:
  # -- Enable gathering Kubernetes Pod logs.
  # @section -- Features - Application Observability
  enabled: true
  destinations: [otlp, otlp-grafana-cloud]
  receivers:
    otlp:
      grpc:
        enabled: true

alloy-metrics:
  enabled: true
  liveDebugging:
    enabled: true
  alloy:
    enableReporting: false
    stabilityLevel: experimental
  controller:
    podAnnotations: {kubernetes.azure.com/set-kube-service-host-fqdn: "true"}

alloy-singleton:
  enabled: true
  liveDebugging:
    enabled: true
  controller:
    podAnnotations: {kubernetes.azure.com/set-kube-service-host-fqdn: "true"}
  alloy:
    enableReporting: false
    stabilityLevel: experimental

alloy-logs:
  enabled: true
  liveDebugging:
    enabled: true
  alloy:
    enableReporting: false
    stabilityLevel: experimental
  controller:
    podAnnotations: {kubernetes.azure.com/set-kube-service-host-fqdn: "true"}

# An Alloy instance for opening receivers to collect application data.
alloy-receiver:
  # -- Deploy the Alloy instance for opening receivers to collect application data.
  # @section -- Collectors - Alloy Receiver
  enabled: true
  liveDebugging:
    enabled: true
  controller:
    podAnnotations: {kubernetes.azure.com/set-kube-service-host-fqdn: "true"}
  extraConfig: |-
      faro.receiver "integrations_app_agent_receiver" {
        server {
          listen_address           = "0.0.0.0"
          listen_port              = 8027
          cors_allowed_origins     = ["*"]
          max_allowed_payload_size = "10MiB"

          rate_limiting {
            rate = 100
          }
        }

        output {
          logs   = [otelcol.receiver.loki.otlp.receiver]
          traces = [otelcol.processor.transform.otlp.input]
        }
      }
  alloy:
    enableReporting: false
    stabilityLevel: experimental
    extraPorts:
      - name: otlp-grpc
        port: 4317
        targetPort: 4317
        protocol: TCP
      - name: faro
        port: 8027
        targetPort: 8027
        protocol: TCP