Trace to metrics. Have no service-graphs on my metric storage

[andreivaloshyn]

Hello!
I use grafana 9.2.0
VictoriaMetrics (VM) to store metrics
Tempo deployed with helm tempo-distributed-0.26.5
As a test app I use synthetic-load-generator which is a part of Tempo examples.

My helm value file

./values.yaml
     1	---
     2	serviceAccount:
     3	  create: true
     4	  name: grafana-tempo
     5	  annotations:
     6	    eks.amazonaws.com/role-arn: "arn:aws:iam::XXXX:role/test01-tempo-grafana-tempo"
     7	
     8	global_overrides:
     9	  max_search_bytes_per_trace: 0
    10	  per_tenant_override_config: /conf/overrides.yaml
    11	  metrics_generator_disable_collection: false
    12	  metrics_generator_processors:
    13	    - service-graphs
    14	    - span-metrics
    15	
    16	fullnameOverride: tempo
    17	rbac:
    18	  create: true
    19	traces:
    20	  jaeger:
    21	    thriftHttp:
    22	      enabled: true
    23	
    24	metricsGenerator:
    25	  enabled: true
    26	  replicas: 1
    27	  nodeSelector:
    28	    type: system
    29	    kubernetes.io/os: linux
    30	    kubernetes.io/arch: amd64
    31	  resources:
    32	    requests:
    33	      cpu: 100m
    34	      memory: 1Gi
    35	    limits:
    36	      cpu: 300m
    37	      memory: 2Gi
    38	  config:
    39	    storage:
    40	      path: /var/tempo/wal
    41	      remote_write_flush_deadline: 1m
    42	      remote_write:
    43	        - url: http://victoria-metrics-cluster-vminsert.victoria-metrics-cluster:8480/insert/3t45t4454534/prometheus/api/v1/write
    44	          remote_timeout: 30s
    45	          name: victoria-metrics
    46	          follow_redirects: true
    47	          queue_config:
    48	            capacity: 10000
    49	            max_shards: 150
    50	            min_shards: 50
    51	            max_samples_per_send: 2000
    52	            batch_send_deadline: 5s
    53	            min_backoff: 30ms
    54	            max_backoff: 5s
    55	          metadata_config:
    56	            send: true
    57	            send_interval: 1m
    58	            max_samples_per_send: 500
    59	
    60	memcached:
    61	  enabled: true
    62	  replicas: 1
    63	  resources:
    64	    requests:
    65	      cpu: 100m
    66	      memory: 1Gi
    67	    limits:
    68	      cpu: 300m
    69	      memory: 2Gi
    70	  nodeSelector:
    71	    type: system
    72	    kubernetes.io/os: linux
    73	    kubernetes.io/arch: amd64
    74	
    75	ingester:
    76	  persistence:
    77	    enabled: true
    78	    size: 20Gi
    79	    storageClass: aws-ebs-csi-retain
    80	  nodeSelector:
    81	    type: system
    82	    kubernetes.io/os: linux
    83	    kubernetes.io/arch: amd64
    84	  resources:
    85	    requests:
    86	      cpu: 500m
    87	      memory: 1Gi
    88	    limits:
    89	      cpu: 1000m
    90	      memory: 2Gi
    91	
    92	distributor:
    93	  autoscaling:
    94	    enabled: true
    95	    minReplicas: 1
    96	    maxReplicas: 10
    97	    targetCPUUtilizationPercentage: 80
    98	    targetMemoryUtilizationPercentage: 70
    99	  replicas: 1
   100	  resources:
   101	    requests:
   102	      cpu: 300m
   103	      memory: 1Gi
   104	    limits:
   105	      cpu: 500m
   106	      memory: 2Gi
   107	  nodeSelector:
   108	    type: system
   109	    kubernetes.io/os: linux
   110	    kubernetes.io/arch: amd64
   111	
   112	compactor:
   113	  replicas: 1
   114	  resources:
   115	    requests:
   116	      cpu: 2000m
   117	      memory: 3Gi
   118	    limits:
   119	      cpu: 4000m
   120	      memory: 3Gi
   121	  nodeSelector:
   122	    type: system
   123	    kubernetes.io/os: linux
   124	    kubernetes.io/arch: amd64
   125	
   126	querier:
   127	  replicas: 2
   128	  resources:
   129	    requests:
   130	      cpu: 100m
   131	      memory: 1Gi
   132	    limits:
   133	      cpu: 300m
   134	      memory: 2Gi
   135	  nodeSelector:
   136	    type: system
   137	    kubernetes.io/os: linux
   138	    kubernetes.io/arch: amd64
   139	
   140	queryFrontend:
   141	  replicas: 1
   142	  resources:
   143	    requests:
   144	      cpu: 100m
   145	      memory: 128Mi
   146	    limits:
   147	      cpu: 300m
   148	      memory: 1Gi
   149	  nodeSelector:
   150	    type: system
   151	    kubernetes.io/os: linux
   152	    kubernetes.io/arch: amd64
   153	
   154	config: |
   155	  query_frontend:
   156	    search:
   157	      max_duration: 0
   158	  multitenancy_enabled: false
   159	  search_enabled: true
   160	  metrics_generator_enabled: {{ .Values.metricsGenerator.enabled }}
   161	  {{- if .Values.metricsGenerator.enabled }}
   162	  metrics_generator:
   163	    ring:
   164	      kvstore:
   165	        store: memberlist
   166	    processor:
   167	      {{- toYaml .Values.metricsGenerator.config.processor | nindent 6 }}
   168	    storage:
   169	      {{- toYaml .Values.metricsGenerator.config.storage | nindent 6 }}
   170	    registry:
   171	      {{- toYaml .Values.metricsGenerator.config.registry | nindent 6 }}
   172	  {{- end }}
   173	  compactor:
   174	    compaction:
   175	      block_retention: 336h
   176	  distributor:
   177	    log_received_traces: true
   178	    log_received_spans:
   179	      enabled: true
   180	      include_all_attributes: true
   181	      filter_by_status_error: true
   182	    receivers:
   183	      jaeger:
   184	        protocols:
   185	          thrift_compact:
   186	            endpoint: 0.0.0.0:6831
   187	          thrift_binary:
   188	            endpoint: 0.0.0.0:6832
   189	          thrift_http:
   190	            endpoint: 0.0.0.0:14268
   191	          grpc:
   192	            endpoint: 0.0.0.0:14250
   193	      otlp:
   194	        protocols:
   195	          http:
   196	            endpoint: 0.0.0.0:55681
   197	          grpc:
   198	            endpoint: 0.0.0.0:4317
   199	  querier:
   200	    frontend_worker:
   201	      frontend_address: tempo-query-frontend-discovery:9095
   202	  ingester:
   203	    complete_block_timeout: 10m
   204	    lifecycler:
   205	      ring:
   206	        replication_factor: 3
   207	        kvstore:
   208	          store: memberlist
   209	      tokens_file_path: /var/tempo/tokens.json
   210	  memberlist:
   211	    abort_if_cluster_join_fails: false
   212	    join_members:
   213	      - tempo-gossip-ring
   214	  overrides:
   215	    {{- toYaml .Values.global_overrides | nindent 2 }}
   216	  server:
   217	    http_listen_port: 3100
   218	    log_level: debug
   219	    grpc_server_max_recv_msg_size: 16777216
   220	    grpc_server_max_send_msg_size: 16777216
   221	  storage:
   222	    trace:
   223	      backend: s3
   224	      s3:
   225	        bucket: digitaltest01-grafana-tempo
   226	        endpoint: s3.amazonaws.com
   227	        insecure: false
   228	      blocklist_poll: 5m
   229	      cache: memcached
   230	      wal:
   231	        path: /var/tempo/wal
   232	      memcached:
   233	        consistent_hash: true
   234	        addresses: "tempo-memcached:11211"
   235	...

As described https://grafana.com/docs/tempo/latest/metrics-generator/service_graphs/#tempo I added

    12	  metrics_generator_processors:
    13	    - service-graphs
    14	    - span-metrics

Log from metric generator

level=debug ts=2022-10-14T13:29:42.543542632Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=103.322µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.561597512Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=32.621µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.563266389Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=65.651µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.566520971Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=58.121µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.570449875Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=71.411µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.602137044Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=38.971µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.604830097Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=65.231µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.607251406Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=58.761µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.613390855Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=38.51µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.613666219Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=35.291µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.614509723Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=35.63µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.616110188Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=54.011µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.621277061Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=28.54µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.628083431Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=64.721µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.630361968Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=36.071µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.634925101Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=62.821µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.641849162Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=39.771µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.643186954Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=86.911µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.643293905Z caller=grpc_logging.go:46 duration=33.88µs method=/tempopb.MetricsGenerator/PushSpans msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.654999944Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=30.271µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.658257336Z caller=grpc_logging.go:46 duration=29.071µs method=/tempopb.MetricsGenerator/PushSpans msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.691464399Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=34.56µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.692900093Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=60.151µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.695040487Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=71.681µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.712064961Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=32.75µs msg="gRPC (success)"
level=debug ts=2022-10-14T13:29:42.738912042Z caller=grpc_logging.go:46 method=/tempopb.MetricsGenerator/PushSpans duration=38.521µs msg="gRPC (success)"

What I see in VM

And Tempo Explorer

I expected Service Graph to work in grafana explorer and it will be nice to have traces_service_graph-* metrics in VM.

Guys, could you please help and point me what I should add or what I'm doing wrong this case.

Thank you!

[joe-elliott]

It's been awhile since I've looked closely at the synthetic load generator. It's possible it doesn't build traces in a way that Tempo recognizes for building service graphs.

Some details are here:
https://grafana.com/docs/tempo/latest/metrics-generator/service_graphs/#how-they-work

The first thing I would check is to make sure that the spans that cross service boundaries have the correct span.kind. They should be client and server.

[andreivaloshyn]

Now it's clear. Thank you!