From 226e9f101aab2faf32b2d6b0c93f6e5832782c25 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 6 Jan 2025 11:50:45 +0100 Subject: [PATCH] feat: add kafka development environment (#15603) --- .gitignore | 1 + tools/dev/kafka/README.md | 124 ++++++++++++++++++ tools/dev/kafka/docker-compose.yaml | 53 ++++++++ tools/dev/kafka/loki-local-config.debug.yaml | 77 +++++++++++ .../provisioning/datasources/default.yml | 11 ++ 5 files changed, 266 insertions(+) create mode 100644 tools/dev/kafka/README.md create mode 100644 tools/dev/kafka/docker-compose.yaml create mode 100644 tools/dev/kafka/loki-local-config.debug.yaml create mode 100644 tools/dev/kafka/provisioning/datasources/default.yml diff --git a/.gitignore b/.gitignore index 5eedfae07d6f4..754c579d69387 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ dist .idea pkg/loki/wal tools/lambda-promtail/main +tools/dev/kafka/data/ # Submodule added by `act` CLI _shared-workflows-dockerhub-login diff --git a/tools/dev/kafka/README.md b/tools/dev/kafka/README.md new file mode 100644 index 0000000000000..e2d60e549cd47 --- /dev/null +++ b/tools/dev/kafka/README.md @@ -0,0 +1,124 @@ +# Loki Kafka Development Setup + +This directory contains the development environment for testing Loki with Kafka integration. The setup provides supporting services (Kafka, Grafana, and a log generator) via docker-compose, while Loki itself needs to be run manually from source code for development purposes. + +## Quick Start + +1. Start the supporting services (Kafka, Grafana, Log Generator): + ```bash + docker-compose up -d + ``` + +2. Run Loki manually with Kafka configuration: + ```bash + # From the root of the Loki repository + go run ./cmd/loki/main.go --config.file=tools/dev/kafka/loki-local-config.debug.yaml --log.level=debug -target=all + ``` + + Note: Loki is not included in docker-compose as it's intended to be run directly from source code for development. + +## Services + +### Kafka +- Broker accessible at `localhost:9092` +- Uses KRaft (no ZooKeeper required) +- Single broker setup for development +- Topic `loki` is used for log ingestion + +### Kafka UI +- Web interface available at http://localhost:8080 +- Monitor topics, messages, and consumer groups +- No authentication required + +### Grafana +- Available at http://localhost:3000 +- Anonymous access enabled (Admin privileges) +- Pre-configured with Loki data source +- Features enabled: + - Loki logs dataplane + - Explore logs shard splitting + - Loki explore app + +### Log Generator +- Automatically sends sample logs to Loki +- Useful for testing and development +- Configured to push logs directly to Loki's HTTP endpoint + +## Configuration Files + +- `docker-compose.yaml`: Service definitions and configuration +- `loki-local-config.debug.yaml`: Loki configuration with Kafka enabled + - Kafka ingestion enabled + - Local storage in `/tmp/loki` + - Debug logging enabled + +## Debugging + +### VSCode Configuration +Create `.vscode/launch.json` in the root directory: +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Loki (Kafka)", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/loki/main.go", + "args": [ + "--config.file=../../tools/dev/kafka/loki-local-config.debug.yaml", + "--log.level=debug", + "-target=all" + ], + "buildFlags": "-mod vendor" + } + ] +} +``` + +## Common Tasks + +### View Logs +```bash +# All services +docker-compose logs -f + +# Specific service +docker-compose logs -f kafka +docker-compose logs -f grafana +``` + +### Reset Environment +```bash +# Stop and remove containers +docker-compose down + +# Remove local Loki data +rm -rf /tmp/loki + +# Start fresh +docker-compose up -d +``` + +### Verify Setup +1. Check Kafka UI (http://localhost:8080) for: + - Broker health + - Topic creation + - Message flow + +2. Check Grafana (http://localhost:3000): + - Navigate to Explore + - Select Loki data source + - Query logs using LogQL + +## Troubleshooting + +- **Loki fails to start**: Check if port 3100 is available +- **No logs in Grafana**: + - Verify Kafka topics are created + - Check log generator is running + - Verify Loki is receiving data through Kafka UI +- **Kafka connection issues**: + - Ensure broker is running (`docker-compose ps`) + - Check broker logs (`docker-compose logs broker`) diff --git a/tools/dev/kafka/docker-compose.yaml b/tools/dev/kafka/docker-compose.yaml new file mode 100644 index 0000000000000..43771a8e9984c --- /dev/null +++ b/tools/dev/kafka/docker-compose.yaml @@ -0,0 +1,53 @@ +services: + grafana: + image: grafana/grafana-enterprise:latest + environment: + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_BASIC_ENABLED=false + - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane exploreLogsShardSplitting + - GF_INSTALL_PLUGINS=https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app + ports: + - 3000:3000/tcp + volumes: + - ./provisioning:/etc/grafana/provisioning/ + - ./data/grafana/:/var/lib/grafana/ + extra_hosts: + - 'host.docker.internal:host-gateway' + kafka-ui: + image: provectuslabs/kafka-ui:latest + ports: + - 8080:8080 + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker:29092 + KAFKA_CLUSTERS_0_METRICS_PORT: 9997 + depends_on: + - broker + extra_hosts: + - 'host.docker.internal:host-gateway' + broker: + image: apache/kafka:latest + hostname: broker + container_name: broker + ports: + - 9092:9092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_NODE_ID: 1 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker:29093 + KAFKA_LISTENERS: PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + generator: + image: ctovena/log-generator:latest + command: -url http://host.docker.internal:3100/loki/api/v1/push diff --git a/tools/dev/kafka/loki-local-config.debug.yaml b/tools/dev/kafka/loki-local-config.debug.yaml new file mode 100644 index 0000000000000..59394cd7b628e --- /dev/null +++ b/tools/dev/kafka/loki-local-config.debug.yaml @@ -0,0 +1,77 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + instance_id: local + kvstore: + store: inmemory + +kafka_config: + topic: "loki" + +querier: + query_partition_ingesters: true + +ingester: + kafka_ingestion: + enabled: true + +distributor: + kafka_writes_enabled: true + ingester_writes_enabled: false + +query_range: + results_cache: + cache: + embedded_cache: + enabled: false + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/tools/dev/kafka/provisioning/datasources/default.yml b/tools/dev/kafka/provisioning/datasources/default.yml new file mode 100644 index 0000000000000..964020e9d0172 --- /dev/null +++ b/tools/dev/kafka/provisioning/datasources/default.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: gdev-testdata + isDefault: true + type: testdata + - name: gdev-loki + type: loki + uid: gdev-loki + access: proxy + url: http://host.docker.internal:3100