[Ingester] Create one goroutine per tenant to flush traces to disk

CHANGELOG.md

@@ -89,6 +89,7 @@ querier:
 * [ENHANCEMENT] Reuse generator code to better refuse "too large" traces. [#4365](https://github.com/grafana/tempo/pull/4365) (@joe-elliott)
   This will cause the ingester to more aggressively and correctly refuse traces. Also added two metrics to better track bytes consumed per tenant in the ingester.
   `tempo_metrics_generator_live_trace_bytes` and `tempo_ingester_live_trace_bytes`.
+* [ENHANCEMENT] Prevent queries in the ingester from blocking flushing traces to disk and memory spikes. [#4483](https://github.com/grafana/tempo/pull/4483) (@joe-elliott)
 * [ENHANCEMENT] Update Azurite image. [#4298](https://github.com/grafana/tempo/pull/4464) (@javiermolinar)
 * [ENHANCEMENT] Update golang.org/x/crypto [#4474](https://github.com/grafana/tempo/pull/4474) (@javiermolinar)
 * [BUGFIX] Handle invalid TraceQL query filter in tag values v2 disk cache [#4392](https://github.com/grafana/tempo/pull/4392) (@electron0zero)

example/tk/jsonnetfile.json

@@ -72,6 +72,15 @@
         }
       },
       "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git",
+          "subdir": "1.30"
+        }
+      },
+      "version": "main"
     }
   ],
   "legacyImports": true

example/tk/jsonnetfile.lock.json

@@ -8,7 +8,7 @@
           "subdir": "grafana"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "Y5nheroSOIwmE+djEVPq4OvvTxKenzdHhpEwaR3Ebjs="
     },
     {
@@ -18,8 +18,8 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
-      "sum": "B49EzIY2WZsFxNMJcgRxE/gcZ9ltnS8pkOOV6Q5qioc="
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
+      "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo="
     },
     {
       "source": {
@@ -28,7 +28,7 @@
           "subdir": "ksonnet-util"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "0y3AFX9LQSpfWTxWKSwoLgbt0Wc9nnCwhMH2szKzHv0="
     },
     {
@@ -38,7 +38,7 @@
           "subdir": "kube-state-metrics/"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "q1YzD+I4InDdfQP6k93W9Lw5U1jpll3gVaS9rUzLR7U="
     },
     {
@@ -48,7 +48,7 @@
           "subdir": "memcached"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "Cc715Y3rgTuimgDFIw+FaKzXSJGRYwt1pFTMbdrNBD8="
     },
     {
@@ -58,7 +58,7 @@
           "subdir": "prometheus"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "b4scQI+UtFxkVgzzD3dfbPMwI7EVtyjv1uETEmizM8M="
     },
     {
@@ -68,7 +68,7 @@
           "subdir": "tanka-util"
         }
       },
-      "version": "84e49c8549fa472c963862f233422c8b368afabe",
+      "version": "e13a89c9d775bc253a559df9bb693be07b5ef65e",
       "sum": "ShSIissXdvCy1izTCDZX6tY7qxCoepE5L+WJ52Hw7ZQ="
     },
     {
@@ -88,8 +88,18 @@
           "subdir": "1.29"
         }
       },
-      "version": "bf9a62cfd32a58c071b8410bfcdec058475dd25e",
+      "version": "6ecbb7709baf27f44b2e48f3529741ae6754ae6a",
       "sum": "i2w3hGbgQmaB73t5LJHSioPOVdYv8ZBvivHiDwZJVyI="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git",
+          "subdir": "1.30"
+        }
+      },
+      "version": "6ecbb7709baf27f44b2e48f3529741ae6754ae6a",
+      "sum": "0OMrWr4xnhN5VWLXwPzCmdcENsifLNAZNzPvrHAnUAo="
     }
   ],
   "legacyImports": false

example/tk/lib/k.libsonnet

@@ -1 +1 @@
-import '1.21/main.libsonnet'
+import '1.30/main.libsonnet'

example/tk/lib/synthetic-load-generator/main.libsonnet

@@ -1,33 +1,122 @@
 {
   local k = import 'ksonnet-util/kausal.libsonnet',
-  local configMap = k.core.v1.configMap,
-  local container = k.core.v1.container,
-  local volumeMount = k.core.v1.volumeMount,
+
   local deployment = k.apps.v1.deployment,
+  local container = k.core.v1.container,
+  local configMap = k.core.v1.configMap,
+  local envVar = k.core.v1.envVar,
   local volume = k.core.v1.volume,
+  local volumeMount = k.core.v1.volumeMount,
 
-  synthetic_load_generator_configmap:
-    configMap.new('synthetic-load-generator') +
-    configMap.withData({
-      'load-generator.json': importstr './load-generator.json',
-    }),
+  k6_tracing_data_deployment:
+    deployment.new('k6-tracing', 1,
+                   [
+                     $.k6_tracing_container,
+                   ],
+                   {
+                     app: 'k6-tracing',
+                   }) +
+    deployment.mixin.spec.template.metadata.withAnnotations({
+      config_hash: std.md5(std.toString($.k6_tracing_config_map.data['template.js'])),
+    }) +
+    deployment.mixin.spec.template.spec.withVolumes([
+      volume.fromConfigMap('k6-tracing', 'k6-tracing'),
+    ]),
 
-  synthetic_load_generator_container::
-    container.new('synthetic-load-gen', 'omnition/synthetic-load-generator:1.0.25') +
-    container.withVolumeMounts([
-      volumeMount.new('conf', '/conf'),
+  k6_tracing_container::
+    container.new('k6-tracing', 'ghcr.io/grafana/xk6-client-tracing:v0.0.5') +
+    container.withArgs([
+      'run',
+      '/var/k6/template.js',
     ]) +
-    container.withEnvMap({
-      TOPOLOGY_FILE: '/conf/load-generator.json',
-      JAEGER_COLLECTOR_URL: 'http://tempo:14268',
-    }),
-
-  synthetic_load_generator_deployment:
-    deployment.new('synthetic-load-generator',
-                   1,
-                   [ $.synthetic_load_generator_container ],
-                   { app: 'synthetic_load_generator' }) +
-    deployment.mixin.spec.template.spec.withVolumes([
-      volume.fromConfigMap('conf', $.synthetic_load_generator_configmap.metadata.name),
+    container.withEnvMixin([envVar.new('ENDPOINT', 'http://distributor:4318')]) +
+    container.withVolumeMountsMixin([
+      volumeMount.withName('k6-tracing')
+      + volumeMount.withMountPath('/var/k6'),
     ]),
+
+
+  tempo_config+:: {
+    storage+: {
+      trace+: {
+        empty_tenant_deletion_age: '24h',
+        empty_tenant_deletion_enabled: true,
+      },
+    },
+  },
+
+  k6_tracing_config_map:
+    configMap.new('k6-tracing') +
+    configMap.withData({
+      'template.js': |||
+        import {sleep} from 'k6';
+        import tracing from 'k6/x/tracing';
+        import { randomIntBetween } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js';
+
+        export const options = {
+            vus: 1,
+            duration: "30m",
+        };
+
+        const endpoint = __ENV.ENDPOINT || "otel-collector:4317"
+        const client = new tracing.Client({
+            endpoint,
+            exporter: tracing.EXPORTER_OTLP_HTTP,
+            tls: {
+              insecure: true,
+            },
+            headers: {
+              'X-Scope-OrgID': '3',
+            },
+        });
+
+        const traceDefaults = {
+            attributeSemantics: tracing.SEMANTICS_HTTP,
+            randomAttributes: {count: 2, cardinality: 5},
+            randomEvents: {count: 0.1, exceptionCount: 0.2, randomAttributes: {count: 6, cardinality: 20}},
+        }
+
+        const traceTemplates = [
+            {
+                defaults: traceDefaults,
+                spans: [
+                    {service: "gateway", name: "/foo", duration: {min: 200, max: 900}, attributes: {"http.status_code": 200, "application.version": "1.1"}},
+                    {service: "gateway", name: "authenticate", duration: {min: 50, max: 100}},
+                    {service: "auth-service", name: "/auth", attributes: {"http.status_code": 200, "application.version": "3.1"}},
+                    {service: "cache", name: "GET", attributes: {"http.status_code": 200, "application.version": "2.1"}},
+                    {service: "identity-service", name: "/user", parentIdx: 2, attributes: {"http.status_code": 200, "application.version": "0.3"}},
+                    {service: "service-A", name: "/id", attributes: {"http.status_code": 200, "application.version": "12.3"}},
+                    {service: "service-B", name: "/id", parentIdx: 4, attributes: {"http.status_code": 200, "application.version": "9.1"}},
+                ]
+            },
+            {
+                defaults: traceDefaults,
+                spans: [
+                    {service: "gateway", name: "/foo", duration: {min: 200, max: 900}, attributes: {"http.status_code": 500}},
+                    {service: "gateway", name: "authenticate", duration: {min: 50, max: 100}},
+                    {service: "auth-service", name: "/auth", attributes: {"http.status_code": 500}},
+                    {service: "cache", name: "GET", attributes: {"http.status_code": 200, "application.version": "2.1"}},
+                    {service: "identity-service", name: "/user", parentIdx: 2, attributes: {"http.status_code": 400, "application.version": "0.4"}},
+                    {service: "service-A", name: "/id", attributes: {"http.status_code": 200}},
+                    {service: "service-B", name: "/id", parentIdx: 4, attributes: {"http.status_code": 200}},
+                ]
+            },
+        ]
+
+        export default function () {
+            const d = new Date();
+            let minutes = d.getMinutes();
+            let idx = Math.floor(minutes / 5) % traceTemplates.length
+
+            const gen = new tracing.TemplatedGenerator(traceTemplates[idx])
+            client.push(gen.traces())
+
+            sleep(randomIntBetween(1, 5));
+        }
+
+        export function teardown() {
+            client.shutdown();
+        }
+      |||,
+    }),
 }

example/tk/tempo-microservices/main.jsonnet

@@ -26,15 +26,21 @@ minio + metrics + load + tempo {
     },
     distributor+: {
       receivers: {
-        opencensus: null,
-        jaeger: {
+        otlp: {
           protocols: {
-            thrift_http: null,
+            grpc: {
+              endpoint: '0.0.0.0:55680',
+            },
+            http: {
+              endpoint: '0.0.0.0:4318',
+            },
           },
         },
       },
     },
     metrics_generator+: {
+      pvc_size: '5Gi',
+      pvc_storage_class: 'local-path',
       ephemeral_storage_limit_size: '2Gi',
       ephemeral_storage_request_size: '1Gi',
     },
@@ -79,12 +85,14 @@ minio + metrics + load + tempo {
   tempo_distributor_container+::
     k.util.resourcesRequests('500m', '500Mi') +
     container.withPortsMixin([
+      containerPort.new('otlp-http', 4318),
       containerPort.new('opencensus', 55678),
       containerPort.new('jaeger-http', 14268),
     ]),
 
   tempo_ingester_container+::
-    k.util.resourcesRequests('500m', '500Mi'),
+    k.util.resourcesRequests('500m', '500Mi') +
+    container.withImagePullPolicy('IfNotPresent'), // prevents ingester from pulling a new image in case you push a custom grafana/tempo:latest
 
   // clear affinity so we can run multiple ingesters on a single node
   tempo_ingester_statefulset+: {

example/tk/vendor/1.30

@@ -0,0 +1 @@
+github.com/jsonnet-libs/k8s-libsonnet/1.30