docker compose for threat detection

docker-compose-kafka.yml

@@ -0,0 +1,49 @@
+services:
+  zoo1:
+    image: confluentinc/cp-zookeeper:6.2.1
+    restart: on-failure:10
+    hostname: zoo1
+    user: "0"  
+    volumes:
+      - ./data-zoo-data:/var/lib/zookeeper/data
+      - ./data-zoo-logs:/var/lib/zookeeper/log
+      - ./data-zoo-secrets:/etc/zookeeper/secrets          
+    container_name: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: zoo1:2888:3888
+
+  kafka1:
+    image: confluentinc/cp-kafka:6.2.1
+    restart: on-failure:10
+    hostname: kafka1
+    user: "0"
+    ports:
+      - "9092:9092"
+      - "19092:19092"
+      - "29092:29092"
+      - "9999:9999"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://localhost:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CREATE_TOPICS: "akto.api.logs:3:3"
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000
+      KAFKA_LOG_RETENTION_HOURS: 5
+      KAFKA_LOG_SEGMENT_BYTES: 104857600
+      KAFKA_LOG_CLEANER_ENABLE: "true"
+      KAFKA_CLEANUP_POLICY: "delete"
+      KAFKA_LOG_RETENTION_BYTES: 10737418240
+    volumes:
+      - ./data-kafka-data:/var/lib/kafka/data
+      - ./data-kafka-secrets:/etc/kafka/secrets       
+    depends_on:
+      - zoo1

docker-compose-mongo.yml

@@ -8,4 +8,4 @@ services:
     volumes:
       - ./data:/data/db
     ports:
-      - "27017:27017"
+      - "27017:27017"

docker-compose-threat-detection-backend.yml

@@ -0,0 +1,53 @@
+version: '3.3'
+
+services:
+  mongo:
+    container_name: mongo
+    image: mongo
+    restart: on-failure:10
+    volumes:
+      - ./data:/data/db
+    ports:
+      - "27017:27017"
+
+
+  kafka-internal:
+    image: confluentinc/cp-kafka:7.8.0
+    container_name: kafka-internal
+    hostname: kafka-internal
+    user: root
+    ports:
+      - "29092:29092" # PLAINTEXT_HOST listener
+      - "19092:19092" # PLAINTEXT listener
+      - "9093:9093" # CONTROLLER listener
+    environment:
+      # Kafka Node Configuration
+      KAFKA_NODE_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,CONTROLLER://kafka-internal:9093,PLAINTEXT_HOST://0.0.0.0:29092
+
+      # Kafka Roles and Controller Quorum
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-internal:9093
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+
+      # Other Configurations
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+
+    volumes:
+        - ./data-kafka-internal-data:/var/lib/kafka/data
+        - ./data-kafka-internal-secrets:/etc/kafka/secrets
+
+  backend:
+      image: ag60/akto-threat-detection-backend
+      env_file: ./docker-threat-detection-backend.env
+      restart: always
+      ports:
+        - "8980:8980"
+        - "9090:9090"
+      depends_on:
+        - kafka-internal
+        - mongo

docker-compose-threat-detection-client.yml

@@ -0,0 +1,54 @@
+services:
+  postgres:
+    image: postgres:15
+    container_name: postgres-internal
+    hostname: postgres-internal
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: akto
+      POSTGRES_PASSWORD: akto
+      POSTGRES_DB: akto
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  kafka-internal:
+    image: confluentinc/cp-kafka:7.8.0
+    container_name: kafka-internal
+    hostname: kafka-internal
+    user: root
+    ports:
+      - "29092:29092" # PLAINTEXT_HOST listener
+      - "19092:19092" # PLAINTEXT listener
+      - "9093:9093" # CONTROLLER listener
+    environment:
+      # Kafka Node Configuration
+      KAFKA_NODE_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,CONTROLLER://kafka-internal:9093,PLAINTEXT_HOST://0.0.0.0:29092
+
+      # Kafka Roles and Controller Quorum
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-internal:9093
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+
+      # Other Configurations
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+
+    volumes:
+        - ./data-kafka-internal-data:/var/lib/kafka/data
+        - ./data-kafka-internal-secrets:/etc/kafka/secrets
+
+  threat-detection:
+    image: ag60/akto-threat-detection
+    env_file: ./docker-threat-detection.env
+    restart: always
+    depends_on:
+      - kafka-internal
+      - postgres
+
+volumes:
+  postgres_data:

docker-compose-threat-detection-infra.yml

@@ -0,0 +1,55 @@
+services:
+  kafka-central:
+    image: confluentinc/cp-kafka:7.8.0
+    container_name: kafka-central
+    hostname: kafka-central
+    user: root
+    ports:
+      - "9093:9093" # CONTROLLER listener
+      - "9094:9094"
+    environment:
+      # Broker ID
+      KAFKA_BROKER_ID: 1
+
+      # KRaft mode settings
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+
+      # Kafka listeners and advertised listeners
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,LISTENER_DOCKER_EXTERNAL_DIFFHOST://:9094
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092, LISTENER_DOCKER_EXTERNAL_DIFFHOST://10.0.143.202:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+
+      # Specify controller listener names
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+
+      # Log directories
+      KAFKA_LOG_DIRS: /var/lib/kafka/data
+
+      # Minimum in-sync replicas
+      KAFKA_MIN_INSYNC_REPLICAS: 1
+
+      # Offsets topic replication factor
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+      # Transaction state log replication factor and min ISR
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+
+    volumes:
+        - ./data-kafka-central-data:/var/lib/kafka/data
+        - ./data-kafka-central-secrets:/etc/kafka/secrets
+
+  redis-central:
+    image: redis:latest
+    container_name: redis-central
+    hostname: redis-central
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+
+volumes:
+  redis_data:

docker-compose.yml

@@ -29,7 +29,7 @@ services:
       - "29092:29092"
       - "9999:9999"
     environment:
-      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://localhost:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"

docker-threat-detection-backend.env

@@ -0,0 +1,3 @@
+AKTO_MONGO_CONN=mongodb://mongo:27017
+AKTO_THREAT_PROTECTION_MONGO_CONN=mongodb://mongo:27017
+THREAT_EVENTS_KAFKA_BROKER_URL=kafka-internal:19092

docker-threat-detection.env

@@ -0,0 +1,9 @@
+AKTO_MONGO_CONN=
+AKTO_TRAFFIC_KAFKA_BOOTSTRAP_SERVER=
+AKTO_INTERNAL_KAFKA_BOOTSTRAP_SERVER=kafka-internal:19092
+AKTO_THREAT_DETECTION_REDIS_URI=redis://redis1:6379
+AKTO_THREAT_DETECTION_POSTGRES=jdbc:postgresql://postgres-internal:5432/akto
+AKTO_THREAT_DETECTION_POSTGRES_USER=akto
+AKTO_THREAT_DETECTION_POSTGRES_PASSWORD=akto
+AKTO_THREAT_PROTECTION_BACKEND_TOKEN=
+AKTO_THREAT_PROTECTION_BACKEND_URL=https://tbs.akto.io