diff --git a/examples/grafana-agent-auto-instrumentation/java/FastSlow.java b/examples/grafana-agent-auto-instrumentation/java/docker/FastSlow.java similarity index 100% rename from examples/grafana-agent-auto-instrumentation/java/FastSlow.java rename to examples/grafana-agent-auto-instrumentation/java/docker/FastSlow.java diff --git a/examples/grafana-agent-auto-instrumentation/java/README.md b/examples/grafana-agent-auto-instrumentation/java/docker/README.md similarity index 97% rename from examples/grafana-agent-auto-instrumentation/java/README.md rename to examples/grafana-agent-auto-instrumentation/java/docker/README.md index 604d8138da..fb67c675ae 100644 --- a/examples/grafana-agent-auto-instrumentation/java/README.md +++ b/examples/grafana-agent-auto-instrumentation/java/docker/README.md @@ -1,4 +1,4 @@ -# Grafana Agent Java profiling via auto-instrumentation example +# Grafana Agent Java profiling via auto-instrumentation example in Docker This repository provides a practical demonstration of leveraging the Grafana Agent for continuous Java application profiling using Pyroscope in a dockerized environment. It illustrates a seamless approach to profiling Java processes, aiding in performance optimization. diff --git a/examples/grafana-agent-auto-instrumentation/java/config.river b/examples/grafana-agent-auto-instrumentation/java/docker/config.river similarity index 93% rename from examples/grafana-agent-auto-instrumentation/java/config.river rename to examples/grafana-agent-auto-instrumentation/java/docker/config.river index b94edb7180..4d18acfdd2 100644 --- a/examples/grafana-agent-auto-instrumentation/java/config.river +++ b/examples/grafana-agent-auto-instrumentation/java/docker/config.river @@ -4,7 +4,7 @@ logging { } discovery.process "all" { - // join kuberenetes targets with process targets on container_id to have k8s labels + // join kubernetes targets with process targets on container_id to have k8s labels // join = discovery.kubernetes.containers.targets } @@ -53,4 +53,4 @@ pyroscope.write "example" { external_labels = { "env" = "example", } -} \ No newline at end of file +} diff --git a/examples/grafana-agent-auto-instrumentation/java/docker-compose.yml b/examples/grafana-agent-auto-instrumentation/java/docker/docker-compose.yml similarity index 100% rename from examples/grafana-agent-auto-instrumentation/java/docker-compose.yml rename to examples/grafana-agent-auto-instrumentation/java/docker/docker-compose.yml diff --git a/examples/grafana-agent-auto-instrumentation/java/grafana-provisioning/datasources/pyroscope.yml b/examples/grafana-agent-auto-instrumentation/java/docker/grafana-provisioning/datasources/pyroscope.yml similarity index 100% rename from examples/grafana-agent-auto-instrumentation/java/grafana-provisioning/datasources/pyroscope.yml rename to examples/grafana-agent-auto-instrumentation/java/docker/grafana-provisioning/datasources/pyroscope.yml diff --git a/examples/grafana-agent-auto-instrumentation/java/grafana-provisioning/plugins/explore-profiles.yml b/examples/grafana-agent-auto-instrumentation/java/docker/grafana-provisioning/plugins/explore-profiles.yml similarity index 100% rename from examples/grafana-agent-auto-instrumentation/java/grafana-provisioning/plugins/explore-profiles.yml rename to examples/grafana-agent-auto-instrumentation/java/docker/grafana-provisioning/plugins/explore-profiles.yml diff --git a/examples/grafana-agent-auto-instrumentation/java/java.Dockerfile b/examples/grafana-agent-auto-instrumentation/java/docker/java.Dockerfile similarity index 100% rename from examples/grafana-agent-auto-instrumentation/java/java.Dockerfile rename to examples/grafana-agent-auto-instrumentation/java/docker/java.Dockerfile diff --git a/examples/grafana-agent-auto-instrumentation/java/kubernetes/README.md b/examples/grafana-agent-auto-instrumentation/java/kubernetes/README.md new file mode 100644 index 0000000000..3cb5784e0f --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/java/kubernetes/README.md @@ -0,0 +1,47 @@ +# Grafana Alloy Java profiling via auto-instrumentation in Kubernetes + +This repository provides a practical demonstration of leveraging Grafana Alloy for continuous Java application profiling using Pyroscope in Kubernetes. +It illustrates a seamless approach to profiling Java processes, aiding in performance optimization. + +## Overview + +Grafana Alloy automates Java process discovery for profiling, streamlining the setup for applications. It enables precise and targeted profiling configurations through the Grafana Alloy settings. + +Java profiling via Grafana Alloy is based on a few components: +- `discovery.process` for process discovery +- `discovery.kubernetes` for adding Kubernetes labels (namespace, pod, and more) +- `discovery.relabel` for detecting java processes and setting up labels +- `pyroscope.java` for enabling profiling for specific applications +- `pyroscope.write` for writing the profiles data to a remote endpoint + +Refer to the [official documentation](https://grafana.com/docs/pyroscope/latest/configure-client/grafana-agent/java/) for an in-depth understanding and additional configuration options for Java profiling with Grafana Alloy. +Also, check the [Grafana Alloy Components reference](https://grafana.com/docs/alloy/latest/reference/components/) for more details on each used component. + +### async-profiler + +The `pyroscope.java` agent component internally uses the [async-profiler](https://github.com/async-profiler/async-profiler) library. +This approach offers a key advantage over other instrumentation mechanisms in that you can profile applications that are already running without interruptions (code changes, config changes or restarts). + +Under the hood, this is achieved by attaching to the application at a process level and issuing commands to control profiling. + +## Getting started + +To use this example: + +1. Set up a local kubernetes cluster using Kind or a similar tool. +2. Clone this repository and navigate to this example's directory. +3. Create a `pyroscope-java` namespace: + ```shell + kubectl create namespace pyroscope-java + ``` +4. Deploy the manifests: + ```shell + kubectl apply -n pyroscope-java -f . + ``` + +After the deployment is operational, the Grafana Alloy will profile the Java application using the defined configuration. +The example will deploy a Grafana instance in the same cluster, available via the `grafana` service at port 3000. + +## Documentation + +Refer to the [official documentation](https://grafana.com/docs/pyroscope/latest/configure-client/grafana-agent/java/) for an in-depth understanding and additional configuration options for Java profiling with Grafana Alloy. diff --git a/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana-alloy.yaml b/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana-alloy.yaml new file mode 100644 index 0000000000..2ba685c07a --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana-alloy.yaml @@ -0,0 +1,183 @@ +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole # needed for the discovery.kubernetes alloy component +metadata: + name: grafana-alloy-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-alloy + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: grafana-alloy-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-alloy-role +subjects: + - kind: ServiceAccount + name: grafana-alloy + namespace: pyroscope-java + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: grafana-alloy +spec: + selector: + matchLabels: + app: grafana-alloy + template: + metadata: + labels: + app: grafana-alloy + spec: + serviceAccountName: grafana-alloy + containers: + - name: grafana-alloy + image: grafana/alloy + command: + - /bin/alloy + - run + - /etc/agent-config/config.river + - --server.http.listen-addr=0.0.0.0:12345 + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AGENT_MODE + value: flow + ports: + - containerPort: 12345 + volumeMounts: + - name: agent-config + mountPath: /etc/agent-config + securityContext: + privileged: true + runAsGroup: 0 + runAsUser: 0 + capabilities: + add: + - PERFMON + - SYS_PTRACE + - SYS_RESOURCE + - SYS_ADMIN + volumes: + - name: agent-config + configMap: + name: agent-config + hostPID: true + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-config +data: + config.river: | + logging { + level = "debug" + format = "logfmt" + } + + // Discovers all kubernetes pods. + // Relies on serviceAccountName=grafana-alloy in the pod spec for permissions. + discovery.kubernetes "pods" { + role = "pod" + } + + // Discovers all processes running on the node. + // Relies on a security context with elevated permissions for the alloy container (running as root). + // Relies on hostPID=true on the pod spec, to be able to see processes from other pods. + discovery.process "all" { + // Merges kubernetes and process data (using container_id), to attach kubernetes labels to discovered processes. + join = discovery.kubernetes.pods.targets + } + + // Drops non-java processes and adjusts labels. + discovery.relabel "java" { + targets = discovery.process.all.targets + // Drops non-java processes. + rule { + source_labels = ["__meta_process_exe"] + action = "keep" + regex = ".*/java$" + } + // Sets up the service_name using the namespace and container names. + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + target_label = "service_name" + separator = "/" + } + // Sets up kubernetes labels (labels with the __ prefix are ultimately dropped). + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + // Sets up the cluster label. + // Relies on a pod-level annotation with the "cluster_name" name. + // Alternatively it can be set up using external_labels in pyroscope.write. + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_cluster_name"] + target_label = "cluster" + } + } + + // Attaches the Pyroscope profiler to the processes returned by the discovery.relabel component. + // Relies on a security context with elevated permissions for the alloy container (running as root). + // Relies on hostPID=true on the pod spec, to be able to access processes from other pods. + pyroscope.java "java" { + profiling_config { + interval = "15s" + alloc = "512k" + cpu = true + lock = "10ms" + sample_rate = 100 + } + forward_to = [pyroscope.write.local.receiver] + targets = discovery.relabel.java.output + } + + pyroscope.write "local" { + // Send metrics to the locally running Pyroscope instance. + endpoint { + url = "http://pyroscope:4040" + } + external_labels = { + "static_label" = "static_label_value", + } + } +--- diff --git a/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana.yaml b/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana.yaml new file mode 100644 index 0000000000..81968d7961 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/java/kubernetes/grafana.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:latest + env: + - name: GF_INSTALL_PLUGINS + value: grafana-pyroscope-app + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin + - name: GF_AUTH_DISABLE_LOGIN_FORM + value: "true" + ports: + - containerPort: 3000 + volumeMounts: + - name: grafana-provisioning + mountPath: /etc/grafana/provisioning + volumes: + - name: grafana-provisioning + configMap: + name: grafana-provisioning + items: + - key: datasources + path: datasources/datasources.yaml + - key: plugins + path: plugins/plugins.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana +spec: + selector: + app: grafana + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-provisioning +data: + "datasources": | + apiVersion: 1 + datasources: + - uid: local-pyroscope + type: grafana-pyroscope-datasource + name: Pyroscope + url: http://pyroscope:4040 + jsonData: + keepCookies: [pyroscope_git_session] + + "plugins": | + apiVersion: 1 + apps: + - type: grafana-pyroscope-app + jsonData: + backendUrl: http://pyroscope:4040 + secureJsonData: diff --git a/examples/grafana-agent-auto-instrumentation/java/kubernetes/java-fast-slow.yaml b/examples/grafana-agent-auto-instrumentation/java/kubernetes/java-fast-slow.yaml new file mode 100644 index 0000000000..b12f81a880 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/java/kubernetes/java-fast-slow.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: java-app-jar +binaryData: # holds FastSlow.jar from the sibling "Docker" directory, as a base64 encoded binary + jar: | + UEsDBAoAAAgAAD1iQlkAAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAFBLAwQUAAgICAA9YkJZAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803My0xLLS7RDUstKs7Mz7NSMNQz4OXyTczM03XOSSwutlJwSywuCc7JL+flci5KTSxJTdF1qrRSMAKq0zNU0HBNzsksKE5VcEzJLyjJLM3V5OXi5QIAUEsHCJiPQHxXAAAAVgAAAFBLAwQUAAgICAASYkJZAAAAAAAAAAAAAAAADgAAAEZhc3RTbG93LmNsYXNznVTLUhNBFD0dkgwMAxke8ohPMEgCSkDxGYRSCkqsoItQaJWrzqSBgclMaqaHxwf4I25csXAlsvADLD/HtXp7QggWgpSpSvfcvvece27fW/3t5+FXAM8wpyOGFg1xAwkkGcxNvs3zDnfX86/Lm8KSDMkZ27XlLENLNreqoxVtGnQD7TAYWhd5IEuOt0PeNbvMEM8u5V7q6ERKg2mgC90M1yLOUNpO3vJcK/R94cr8wq6wQun5AUO/K3ZKtrvuiJUNX/BKw8WQzeaK56JLwt+2LVHopmp6DVxCH2mwuOMwDJ2FnSc3Lzui0I4BDGpIG7iMKwyZi2Si+wjCctWmi5nN/oP/jPyLoQx9USCh816FCFNF2xWvwmpZ+CsKx9BRktzaWua1Izte5bbL0Jd9V2z2pyR9urNCbpVBX9i1RE3anhtoyDCMnlMJBR1Ha7jVaE9EueRK4fthTYrKcQypcXi1XOEZJSIzydDbuNgTY1LQMMbQ0zw+gddLXuhbYtGOSmtMzISKpYF77nkykD6vLQu54VUCE9lUEpNq0KYMjEbWPR3TuK/hgYGHeMQw3Mxju9velsgXI4lEwde4Rd3aY2ivnrS+Z4unQPWML7hbcUSQKXreVlgrnL7hs4ArezXxf856yvOxudNeNVglW6rRMZZcV/jzDg8CQT2nk5ELlafh6R8d/1sojXg9GEPUhBjULw6mHghar5J1lXZGe2LsM9gn+iBOWpPRYQva1ONwFDoXQYG+dOIDkun3++l45QBaOqFWvk+eWITtJJxiSEBTTwyuR0xMfd4gHbGIbIp2pafrAB3F8UP0AF/QH8ObpoY6Twdl7YSJXgxHrDHcxIipq3E6opo+KixlpknIW3NArR9//TgWpEcBJoG7GmIQsal6cmSOR6G3cYd2Ve1dOptAnv6PIzjDE8xgFoO/AVBLBwipPrnIuQIAAG0FAABQSwECCgAKAAAIAAA9YkJZAAAAAAAAAAAAAAAACQAEAAAAAAAAAAAAAAAAAAAATUVUQS1JTkYv/soAAFBLAQIUABQACAgIAD1iQlmYj0B8VwAAAFYAAAAUAAAAAAAAAAAAAAAAACsAAABNRVRBLUlORi9NQU5JRkVTVC5NRlBLAQIUABQACAgIABJiQlmpPrnIuQIAAG0FAAAOAAAAAAAAAAAAAAAAAMQAAABGYXN0U2xvdy5jbGFzc1BLBQYAAAAAAwADALkAAAC5AwAAAAA= +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: java-fast-slow +spec: + replicas: 1 + selector: + matchLabels: + app: java-fast-slow + template: + metadata: + annotations: + cluster-name: dev-us-east-1 + labels: + app: java-fast-slow + spec: + containers: + - name: java-fast-slow + image: openjdk:21-jdk-slim + imagePullPolicy: IfNotPresent + command: [ "java" ] + args: [ "-jar", "/app/FastSlow.jar" ] + volumeMounts: + - name: app-jar + mountPath: /app + volumes: + - name: app-jar + configMap: + name: java-app-jar + items: + - key: jar + path: FastSlow.jar diff --git a/examples/grafana-agent-auto-instrumentation/java/kubernetes/pyroscope.yaml b/examples/grafana-agent-auto-instrumentation/java/kubernetes/pyroscope.yaml new file mode 100644 index 0000000000..6fece4a132 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/java/kubernetes/pyroscope.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pyroscope +spec: + replicas: 1 + selector: + matchLabels: + app: pyroscope + template: + metadata: + labels: + app: pyroscope + spec: + containers: + - name: pyroscope + image: grafana/pyroscope:latest + ports: + - containerPort: 4040 +--- + +apiVersion: v1 +kind: Service +metadata: + name: pyroscope +spec: + selector: + app: pyroscope + ports: + - protocol: TCP + port: 4040 + targetPort: 4040