Skip to content

update documentation #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
maldwg merged 3 commits into from
Jun 30, 2026
Merged

update documentation #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2b2519
update documentation
maldwg Jun 29, 2026
1c38aa1
Add kafka and clickhouse self heal functionality for lost connections
maldwg Jun 30, 2026
944ab75
overhaul gpu docker compose files
maldwg Jun 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ pipeline:
backend: auto
batch_size: auto

resilience:
retry:
initial_delay_seconds: 1.0
max_delay_seconds: 30.0
backoff_multiplier: 2.0
jitter_seconds: 0.25
log_every_attempts: 5

scaling:
defaults:
executor: thread
Expand Down
110 changes: 10 additions & 100 deletions docker/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ services:
condition: service_healthy
kafka3:
condition: service_healthy
profiles: ["dev"]
profiles: ["dev", "dev-gpu"]

inspector:
extends:
Expand All @@ -190,71 +190,7 @@ services:
condition: service_healthy
kafka3:
condition: service_healthy
profiles: ["prod"]

inspector-dev-gpu:
extends:
file: "docker-compose/dev/docker-compose.pipeline.yml"
service: inspector
depends_on:
kafka1:
condition: service_healthy
kafka2:
condition: service_healthy
kafka3:
condition: service_healthy
deploy:
mode: "replicated"
replicas: 1
resources:
limits:
cpus: '2'
memory: 512m
reservations:
cpus: '1'
memory: 256m
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_inspection
- NUMBER_OF_INSTANCES=1
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
profiles: ["dev-gpu"]

inspector-gpu:
extends:
file: "docker-compose/prod/docker-compose.pipeline.yml"
service: inspector
depends_on:
kafka1:
condition: service_healthy
kafka2:
condition: service_healthy
kafka3:
condition: service_healthy
deploy:
mode: "replicated"
replicas: 1
resources:
limits:
cpus: '2'
memory: 512m
reservations:
cpus: '1'
memory: 256m
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_inspection
- NUMBER_OF_INSTANCES=1
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
profiles: ["prod-gpu"]
profiles: ["prod", "prod-gpu"]


detector-dev:
Expand All @@ -270,22 +206,22 @@ services:
condition: service_healthy
profiles: ["dev"]

detector:
detector-dev-gpu:
extends:
file: "docker-compose/prod/docker-compose.pipeline.yml"
service: detector
file: "docker-compose/dev/docker-compose.pipeline.yml"
service: detector-gpu
depends_on:
kafka1:
condition: service_healthy
kafka2:
condition: service_healthy
kafka3:
condition: service_healthy
profiles: ["prod"]
profiles: ["dev-gpu"]

detector-dev-gpu:
detector:
extends:
file: "docker-compose/dev/docker-compose.pipeline.yml"
file: "docker-compose/prod/docker-compose.pipeline.yml"
service: detector
depends_on:
kafka1:
Expand All @@ -294,45 +230,19 @@ services:
condition: service_healthy
kafka3:
condition: service_healthy
deploy:
mode: "replicated"
replicas: 1
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_analysis
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
profiles: ["dev-gpu"]
profiles: ["prod"]

detector-gpu:
extends:
file: "docker-compose/prod/docker-compose.pipeline.yml"
service: detector
service: detector-gpu
depends_on:
kafka1:
condition: service_healthy
kafka2:
condition: service_healthy
kafka3:
condition: service_healthy
deploy:
mode: "replicated"
replicas: 1
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_analysis
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
profiles: ["prod-gpu"]

alerter-dev:
Expand Down
12 changes: 12 additions & 0 deletions docker/docker-compose/base/docker-compose.kafka.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ services:
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
KAFKA_LOG_RETENTION_HOURS: 4
KAFKA_LOG_RETENTION_BYTES: 10737418240
KAFKA_LOG_SEGMENT_BYTES: 1073741824
KAFKA_LOG_CLEANUP_POLICY: delete
volumes:
- kafka-data1:/var/lib/kafka/data

Expand Down Expand Up @@ -80,6 +84,10 @@ services:
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
KAFKA_LOG_RETENTION_HOURS: 4
KAFKA_LOG_RETENTION_BYTES: 10737418240
KAFKA_LOG_SEGMENT_BYTES: 1073741824
KAFKA_LOG_CLEANUP_POLICY: delete
volumes:
- kafka-data2:/var/lib/kafka/data

Expand Down Expand Up @@ -113,6 +121,10 @@ services:
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
KAFKA_LOG_RETENTION_HOURS: 4
KAFKA_LOG_RETENTION_BYTES: 10737418240
KAFKA_LOG_SEGMENT_BYTES: 1073741824
KAFKA_LOG_CLEANUP_POLICY: delete
volumes:
- kafka-data3:/var/lib/kafka/data

Expand Down
23 changes: 23 additions & 0 deletions docker/docker-compose/dev/docker-compose.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,29 @@ services:
environment:
- GROUP_ID=data_analysis

detector-gpu:
build:
context: ../../..
dockerfile: docker/dockerfiles/Dockerfile.detector
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
networks:
hamstring:
deploy:
mode: "replicated"
replicas: 1
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_analysis
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility

alerter:
build:
context: ../../..
Expand Down
2 changes: 1 addition & 1 deletion docker/docker-compose/prod/docker-compose.monitoring.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
services:
monitoring_agent:
image: ghcr.io/astraos-de/hamstring-monitoring:v2.0.0
image: ghcr.io/astraos-de/hamstring-monitoring:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand Down
33 changes: 27 additions & 6 deletions docker/docker-compose/prod/docker-compose.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
services:
logserver:
image: ghcr.io/astraos-de/hamstring-logserver:v2.0.0
image: ghcr.io/astraos-de/hamstring-logserver:v2.2.0-dev
restart: "unless-stopped"
networks:
hamstring:
Expand All @@ -10,7 +10,7 @@ services:
environment:
- GROUP_ID=log_storage
logcollector:
image: ghcr.io/astraos-de/hamstring-logcollector:v2.0.0
image: ghcr.io/astraos-de/hamstring-logcollector:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand All @@ -20,7 +20,7 @@ services:
- GROUP_ID=log_collection

prefilter:
image: ghcr.io/astraos-de/hamstring-prefilter:v2.0.0
image: ghcr.io/astraos-de/hamstring-prefilter:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand All @@ -33,7 +33,7 @@ services:
- GROUP_ID=log_filtering

inspector:
image: ghcr.io/astraos-de/hamstring-inspector:v2.0.0
image: ghcr.io/astraos-de/hamstring-inspector:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand All @@ -54,7 +54,7 @@ services:
- NUMBER_OF_INSTANCES=1

detector:
image: ghcr.io/astraos-de/hamstring-detector:v2.0.0
image: ghcr.io/astraos-de/hamstring-detector:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand All @@ -66,8 +66,29 @@ services:
environment:
- GROUP_ID=data_analysis

detector-gpu:
image: ghcr.io/astraos-de/hamstring-detector:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
networks:
hamstring:
deploy:
mode: "replicated"
replicas: 1
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
environment:
- GROUP_ID=data_analysis
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility

alerter:
image: ghcr.io/astraos-de/hamstring-alerter:v2.0.0
image: ghcr.io/astraos-de/hamstring-alerter:v2.2.0-dev
restart: "unless-stopped"
volumes:
- ../../../config.yaml:/app/config.yaml
Expand Down
Loading
Loading