feat: add kafka development environment (#15603)

.gitignore

@@ -35,6 +35,7 @@ dist
 .idea
 pkg/loki/wal
 tools/lambda-promtail/main
+tools/dev/kafka/data/
 
 # Submodule added by `act` CLI
 _shared-workflows-dockerhub-login

tools/dev/kafka/README.md

@@ -0,0 +1,124 @@
+# Loki Kafka Development Setup
+
+This directory contains the development environment for testing Loki with Kafka integration. The setup provides supporting services (Kafka, Grafana, and a log generator) via docker-compose, while Loki itself needs to be run manually from source code for development purposes.
+
+## Quick Start
+
+1. Start the supporting services (Kafka, Grafana, Log Generator):
+   ```bash
+   docker-compose up -d
+   ```
+
+2. Run Loki manually with Kafka configuration:
+   ```bash
+   # From the root of the Loki repository
+   go run ./cmd/loki/main.go --config.file=tools/dev/kafka/loki-local-config.debug.yaml --log.level=debug -target=all
+   ```
+
+   Note: Loki is not included in docker-compose as it's intended to be run directly from source code for development.
+
+## Services
+
+### Kafka
+- Broker accessible at `localhost:9092`
+- Uses KRaft (no ZooKeeper required)
+- Single broker setup for development
+- Topic `loki` is used for log ingestion
+
+### Kafka UI
+- Web interface available at http://localhost:8080
+- Monitor topics, messages, and consumer groups
+- No authentication required
+
+### Grafana
+- Available at http://localhost:3000
+- Anonymous access enabled (Admin privileges)
+- Pre-configured with Loki data source
+- Features enabled:
+  - Loki logs dataplane
+  - Explore logs shard splitting
+  - Loki explore app
+
+### Log Generator
+- Automatically sends sample logs to Loki
+- Useful for testing and development
+- Configured to push logs directly to Loki's HTTP endpoint
+
+## Configuration Files
+
+- `docker-compose.yaml`: Service definitions and configuration
+- `loki-local-config.debug.yaml`: Loki configuration with Kafka enabled
+  - Kafka ingestion enabled
+  - Local storage in `/tmp/loki`
+  - Debug logging enabled
+
+## Debugging
+
+### VSCode Configuration
+Create `.vscode/launch.json` in the root directory:
+```json
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Loki (Kafka)",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/loki/main.go",
+            "args": [
+                "--config.file=../../tools/dev/kafka/loki-local-config.debug.yaml",
+                "--log.level=debug",
+                "-target=all"
+            ],
+            "buildFlags": "-mod vendor"
+        }
+    ]
+}
+```
+
+## Common Tasks
+
+### View Logs
+```bash
+# All services
+docker-compose logs -f
+
+# Specific service
+docker-compose logs -f kafka
+docker-compose logs -f grafana
+```
+
+### Reset Environment
+```bash
+# Stop and remove containers
+docker-compose down
+
+# Remove local Loki data
+rm -rf /tmp/loki
+
+# Start fresh
+docker-compose up -d
+```
+
+### Verify Setup
+1. Check Kafka UI (http://localhost:8080) for:
+   - Broker health
+   - Topic creation
+   - Message flow
+
+2. Check Grafana (http://localhost:3000):
+   - Navigate to Explore
+   - Select Loki data source
+   - Query logs using LogQL
+
+## Troubleshooting
+
+- **Loki fails to start**: Check if port 3100 is available
+- **No logs in Grafana**:
+  - Verify Kafka topics are created
+  - Check log generator is running
+  - Verify Loki is receiving data through Kafka UI
+- **Kafka connection issues**:
+  - Ensure broker is running (`docker-compose ps`)
+  - Check broker logs (`docker-compose logs broker`)

tools/dev/kafka/docker-compose.yaml

@@ -0,0 +1,53 @@
+services:
+  grafana:
+    image: grafana/grafana-enterprise:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_BASIC_ENABLED=false
+      - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane exploreLogsShardSplitting
+      - GF_INSTALL_PLUGINS=https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
+    ports:
+      - 3000:3000/tcp
+    volumes:
+      - ./provisioning:/etc/grafana/provisioning/
+      - ./data/grafana/:/var/lib/grafana/
+    extra_hosts:
+      - 'host.docker.internal:host-gateway'
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - 8080:8080
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker:29092
+      KAFKA_CLUSTERS_0_METRICS_PORT: 9997
+    depends_on:
+      - broker
+    extra_hosts:
+      - 'host.docker.internal:host-gateway'
+  broker:
+    image: apache/kafka:latest
+    hostname: broker
+    container_name: broker
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker:29093
+      KAFKA_LISTENERS: PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+  generator:
+    image: ctovena/log-generator:latest
+    command: -url http://host.docker.internal:3100/loki/api/v1/push

tools/dev/kafka/loki-local-config.debug.yaml

@@ -0,0 +1,77 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    instance_id: local
+    kvstore:
+      store: inmemory
+
+kafka_config:
+  topic: "loki"
+
+querier:
+  query_partition_ingesters: true
+
+ingester:
+  kafka_ingestion:
+    enabled: true
+
+distributor:
+  kafka_writes_enabled: true
+  ingester_writes_enabled: false
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: false
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

tools/dev/kafka/provisioning/datasources/default.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: gdev-testdata
+    isDefault: true
+    type: testdata
+  - name: gdev-loki
+    type: loki
+    uid: gdev-loki
+    access: proxy
+    url: http://host.docker.internal:3100