From fdf0ec4a73ee27ebdf3fa253fb4fa513b46dc884 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 5 Nov 2026 10:18:36 +0100 Subject: [PATCH 1/3] feat: Collect the containerdebug logs --- CHANGELOG.md | 7 + .../kafka/pages/usage-guide/logging.adoc | 8 + .../src/controller/build/command.rs | 2 + .../src/controller/build/container.rs | 23 +- .../build/properties/product_logging/mod.rs | 2 + .../product_logging/vector-test.yaml | 279 ++++++++++++++++++ .../properties/product_logging/vector.yaml | 87 ++++++ .../controller/build/resource/statefulset.rs | 3 + .../src/controller/build/role_group/broker.rs | 8 +- .../kafka-vector-aggregator-values.yaml.j2 | 54 +++- .../test_log_aggregation.cpython-313.pyc | Bin 0 -> 1670 bytes 11 files changed, 456 insertions(+), 17 deletions(-) create mode 100644 tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 0492b5a38..12b77e5de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. ([#1021]). - Support Kerberos (GSSAPI) authentication on KRaft controllers, covering both broker-to-controller and controller-to-controller (Raft) traffic ([#1024]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#1053]). - Support for Kafka 4.3.1 ([#1022]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1029]). - Brokers now have a default affinity to the OPA Pods when OPA authorization is configured ([#1035]). @@ -71,6 +73,10 @@ All notable changes to this project will be documented in this file. - `spec.clusterConfig.metadataManager` now uses PascalCase values (`ZooKeeper` and `KRaft`) as per Kubernetes conventions. The lowercase values `zookeeper` and `kraft` are deprecated, but are still accepted for backwards compatibility and may be removed in a future release ([#1034]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#1053]). ### Removed @@ -106,6 +112,7 @@ All notable changes to this project will be documented in this file. [#1049]: https://github.com/stackabletech/kafka-operator/pull/1049 [#1051]: https://github.com/stackabletech/kafka-operator/pull/1051 [#1052]: https://github.com/stackabletech/kafka-operator/pull/1052 +[#1053]: https://github.com/stackabletech/kafka-operator/pull/1053 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/usage-guide/logging.adoc b/docs/modules/kafka/pages/usage-guide/logging.adoc index 34787b541..8b1bc2a2e 100644 --- a/docs/modules/kafka/pages/usage-guide/logging.adoc +++ b/docs/modules/kafka/pages/usage-guide/logging.adoc @@ -14,4 +14,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `kafka` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/kafka` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index 6b7879081..2955fce23 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -60,6 +60,7 @@ pub fn broker_kafka_container_commands( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents \"$FILE_LOG_DIRECTORY\" containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & {set_realm_env} @@ -154,6 +155,7 @@ pub fn controller_kafka_container_command( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents \"$FILE_LOG_DIRECTORY\" containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & {set_realm_env} diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 0c109658c..b415ffef2 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -5,6 +5,7 @@ use std::str::FromStr; +use const_format::concatcp; use stackable_operator::{ builder::pod::{container::FieldPathEnvVar, volume::VolumeBuilder}, commons::{ @@ -42,8 +43,13 @@ use crate::{ stackable_operator::constant!(pub(crate) POD_NAME: EnvVarName = "POD_NAME"); stackable_operator::constant!(KAFKA_CLIENT_PORT: EnvVarName = "KAFKA_CLIENT_PORT"); stackable_operator::constant!(EXTRA_ARGS: EnvVarName = "EXTRA_ARGS"); -// Needed for the `containerdebug` process to log its tracing information to. -stackable_operator::constant!(pub(crate) CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); +// Let the `containerdebug` process write its tracing information to a file. These are the generic +// `stackable-telemetry` variables and apply to the whole Kafka container, so any other +// telemetry-based process in it would log into the same directory and the Vector agent would label +// its events with `container: containerdebug`. +stackable_operator::constant!(pub(crate) FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +stackable_operator::constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +stackable_operator::constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); // The env var and the ZooKeeper discovery ConfigMap key holding the ZooKeeper connection // string (the same string, as the env var name is used as the ConfigMap key). @@ -103,10 +109,15 @@ pub(crate) fn common_kafka_env( &KAFKA_LOG4J_OPTS, kafka_log_opts(&resolved_product_image.product_version), ) + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. .with_value( - &CONTAINERDEBUG_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), + &FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), ) + // Rotation is opt-in, so without these the file would grow unbounded. + .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") + .with_value(&FILE_LOG_MAX_FILES, "2") .with_value( &KAFKA_NODE_ID_OFFSET, node_id_hash32_offset(kafka_role, role_group_name.as_ref()).to_string(), @@ -169,7 +180,9 @@ mod tests { let _ = *POD_NAME; let _ = *KAFKA_CLIENT_PORT; let _ = *EXTRA_ARGS; - let _ = *CONTAINERDEBUG_LOG_DIRECTORY; + let _ = *FILE_LOG_DIRECTORY; + let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *ZOOKEEPER; let _ = *ZOOKEEPER_CONFIG_MAP_KEY; } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index 7e428dd0a..603bb045e 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -127,5 +127,7 @@ mod tests { // The two Kafka log formats must be handled ... assert!(content.contains("files_log4j")); assert!(content.contains("files_log4j2")); + assert!(content.contains("files_tracing_rs")); + assert!(content.contains("*.tracing-rs.json")); } } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index 5af208088..a0326bea9 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -75,6 +75,285 @@ tests: "timestamp": t'2025-10-02T09:27:28.582Z' } + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test unparsable tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Test tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Test tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Test tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: kafka-broker-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "kafka", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "kafka-broker-default-0", + "role": "broker", + "roleGroup": "default" + } + assert_eq!(expected_log_event, .) - name: Test Vector internal logs inputs: diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 80bf691c2..924b232b9 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -25,6 +25,12 @@ sources: - ${LOG_DIR}/*/*.log4j2.xml line_delimiter: "\r\n" + # Reads the logs of the containerdebug process + files_tracing_rs: + type: file + include: + - ${LOG_DIR}/*/*.tracing-rs.json + transforms: processed_files_log4j: inputs: @@ -226,10 +232,91 @@ transforms: } } + # Transforms the tracing-rs JSON logs written by the containerdebug process + processed_files_tracing_rs: + inputs: + - files_tracing_rs + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%+") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.target) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger/target not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], upcase(level)) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = upcase(level) + } + + fields, err = object(event.fields) + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { + .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message + } + + if is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + extended_logs_files: inputs: - processed_files_log4j - processed_files_log4j2 + - processed_files_tracing_rs type: remap source: | del(.source_type) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 0a535f054..681e35443 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -167,6 +167,9 @@ fn add_common_pod_config( .context(AddVolumeSnafu)? .add_empty_dir_volume( &*STACKABLE_LOG_DIR_NAME, + // Only the Kafka log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_KAFKA_LOG_FILES_SIZE], )), diff --git a/rust/operator-binary/src/controller/build/role_group/broker.rs b/rust/operator-binary/src/controller/build/role_group/broker.rs index 7b840aad3..d496124dd 100644 --- a/rust/operator-binary/src/controller/build/role_group/broker.rs +++ b/rust/operator-binary/src/controller/build/role_group/broker.rs @@ -403,12 +403,12 @@ fn jaas_config_file(is_kerberos_enabled: bool) -> String { mod tests { use super::*; use crate::controller::{ - build::container::CONTAINERDEBUG_LOG_DIRECTORY, + build::container::FILE_LOG_DIRECTORY, test_support::{broker_role_group_config, minimal_kafka, validated_cluster}, }; /// The user-supplied `envOverrides` must be merged in after all operator-set environment - /// variables, so that they can override any of them. `CONTAINERDEBUG_LOG_DIRECTORY` is used + /// variables, so that they can override any of them. `FILE_LOG_DIRECTORY` is used /// as the example here because it is set unconditionally by the operator. The controller /// builder merges the same [`EnvVarSet`] the same way. #[test] @@ -437,7 +437,7 @@ mod tests { let mut validated_rg = broker_role_group_config(&cluster, "default").clone(); validated_rg.env_overrides = validated_rg .env_overrides - .with_value(&CONTAINERDEBUG_LOG_DIRECTORY, "/custom/log/dir"); + .with_value(&FILE_LOG_DIRECTORY, "/custom/log/dir"); let (builder, _) = build(&cluster, &role_group_name, &validated_rg).expect("the role group builds"); @@ -458,7 +458,7 @@ mod tests { let containerdebug: Vec<_> = env .iter() - .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .filter(|env_var| env_var.name == "FILE_LOG_DIRECTORY") .collect(); assert_eq!( containerdebug.len(), diff --git a/tests/templates/kuttl/logging/kafka-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/kafka-vector-aggregator-values.yaml.j2 index 8cd3ade21..66c8921d9 100644 --- a/tests/templates/kuttl/logging/kafka-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/kafka-vector-aggregator-values.yaml.j2 @@ -22,35 +22,55 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigBrokerKafka: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-kafka-broker-automatic-log-config-0" && .container == "kafka" + filteredAutomaticLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-kafka-broker-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-kafka-broker-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigBrokerKafka: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-kafka-broker-custom-log-config-0" && .container == "kafka" + filteredCustomLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-kafka-broker-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-kafka-broker-custom-log-config-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -58,11 +78,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: diff --git a/tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc b/tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dedecadb7f862e3dacfe1b88b1aab3ee1d8a34b2 GIT binary patch literal 1670 zcmZ`(OK%%h6h7k_KVrwPxK5K0ClgCWxRUKgRZHu>6jgmf9?irEno8DWd~Mu}XRLe2 zff!M;L5j+vszgOtQC0-Ci!NExKY(mZCHIyEsH^Tw(hWP#_!U&awS4Y5_q@Jy?zyw? z_j>`?m*;=I9d!b{w1xf>d(8f2WFCM5itrxL?k-S4aZCVp+!7Tr^C5s50=VUf!-<}g z47a2>DAEa`dmuwmC<+PZD_dO9dRCpygVfHo{|nkafYpI+*)d=TOdQgP;|6B${6s}M z?KEIj>KiP(`rN@Dhb%1HS>9NEY=)2(mVIw1QB<61IMKskII@lid>}GM8W?g{-vTg? z|EHI$A6IePoylW_`Eh`(xa63R0r)@$Lo&owe?Mb(&?Uu_`5Hak0Ea;t>i3ceSqU>P z0T|A#Q}N>Gi5*>1kFr*GY1nss#&m%*tVnx&8!{Oe0m<;P>G8y*qdB>^a;+-gXGaCR_e+D=^Xujr=w454pCkzMh1@$i16) zZ|-=$3w|4H`rp~Q+zKE5X5#+D&ilLZW;nTZ{m)2jUv!MPcqF#1{t*2>`tXb9%)9k7 z*~U!vo^U_Y7Qq>816sWHz;JN8_SJ=*8%^I-UAj=0FEo+*rd~#;w})EMsh!P6G*Lf# zrV%~6?QM-tKlC?7Q`^2je53V=*`{x{6^S(>M;=FZZ$6s-X|9<#hjIOZRw#`K{DFGp zTqAJ~4cZtZyyuDwf%bI)q~YyaQyhOTMSqhH VKbMB;-pOZDtnCD8f*wJie*m~BrmX+~ literal 0 HcmV?d00001 From 7513772c054f59bd840aab99e3daae8da6067b9a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 5 Nov 2026 10:51:11 +0100 Subject: [PATCH 2/3] chore: Remove test artifact --- .../test_log_aggregation.cpython-313.pyc | Bin 1670 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc diff --git a/tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc b/tests/templates/kuttl/smoke-kraft/__pycache__/test_log_aggregation.cpython-313.pyc deleted file mode 100644 index dedecadb7f862e3dacfe1b88b1aab3ee1d8a34b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1670 zcmZ`(OK%%h6h7k_KVrwPxK5K0ClgCWxRUKgRZHu>6jgmf9?irEno8DWd~Mu}XRLe2 zff!M;L5j+vszgOtQC0-Ci!NExKY(mZCHIyEsH^Tw(hWP#_!U&awS4Y5_q@Jy?zyw? z_j>`?m*;=I9d!b{w1xf>d(8f2WFCM5itrxL?k-S4aZCVp+!7Tr^C5s50=VUf!-<}g z47a2>DAEa`dmuwmC<+PZD_dO9dRCpygVfHo{|nkafYpI+*)d=TOdQgP;|6B${6s}M z?KEIj>KiP(`rN@Dhb%1HS>9NEY=)2(mVIw1QB<61IMKskII@lid>}GM8W?g{-vTg? z|EHI$A6IePoylW_`Eh`(xa63R0r)@$Lo&owe?Mb(&?Uu_`5Hak0Ea;t>i3ceSqU>P z0T|A#Q}N>Gi5*>1kFr*GY1nss#&m%*tVnx&8!{Oe0m<;P>G8y*qdB>^a;+-gXGaCR_e+D=^Xujr=w454pCkzMh1@$i16) zZ|-=$3w|4H`rp~Q+zKE5X5#+D&ilLZW;nTZ{m)2jUv!MPcqF#1{t*2>`tXb9%)9k7 z*~U!vo^U_Y7Qq>816sWHz;JN8_SJ=*8%^I-UAj=0FEo+*rd~#;w})EMsh!P6G*Lf# zrV%~6?QM-tKlC?7Q`^2je53V=*`{x{6^S(>M;=FZZ$6s-X|9<#hjIOZRw#`K{DFGp zTqAJ~4cZtZyyuDwf%bI)q~YyaQyhOTMSqhH VKbMB;-pOZDtnCD8f*wJie*m~BrmX+~ From d364dee3fcb80157480482d8d2397f888c6c9a91 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 5 Nov 2026 14:19:50 +0100 Subject: [PATCH 3/3] chore: Update changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b77e5de..fb26661b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ All notable changes to this project will be documented in this file. - Support Kerberos (GSSAPI) authentication on KRaft controllers, covering both broker-to-controller and controller-to-controller (Raft) traffic ([#1024]). - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to - the Vector aggregator ([#1053]). + the Vector aggregator ([#1055]). - Support for Kafka 4.3.1 ([#1022]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1029]). - Brokers now have a default affinity to the OPA Pods when OPA authorization is configured ([#1035]). @@ -76,7 +76,7 @@ All notable changes to this project will be documented in this file. - The `containerdebug` process writes its log file again. The operator set `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is - rotated hourly and at most two files are kept, because rotation is opt-in ([#1053]). + rotated hourly and at most two files are kept, because rotation is opt-in ([#1055]). ### Removed @@ -112,7 +112,7 @@ All notable changes to this project will be documented in this file. [#1049]: https://github.com/stackabletech/kafka-operator/pull/1049 [#1051]: https://github.com/stackabletech/kafka-operator/pull/1051 [#1052]: https://github.com/stackabletech/kafka-operator/pull/1052 -[#1053]: https://github.com/stackabletech/kafka-operator/pull/1053 +[#1055]: https://github.com/stackabletech/kafka-operator/pull/1055 ## [26.7.0] - 2026-07-21