diff --git a/docker-compose-kafka.yml b/docker-compose-kafka.yml new file mode 100644 index 0000000..7d60a78 --- /dev/null +++ b/docker-compose-kafka.yml @@ -0,0 +1,49 @@ +services: + zoo1: + image: confluentinc/cp-zookeeper:6.2.1 + restart: on-failure:10 + hostname: zoo1 + user: "0" + volumes: + - ./data-zoo-data:/var/lib/zookeeper/data + - ./data-zoo-logs:/var/lib/zookeeper/log + - ./data-zoo-secrets:/etc/zookeeper/secrets + container_name: zoo1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zoo1:2888:3888 + + kafka1: + image: confluentinc/cp-kafka:6.2.1 + restart: on-failure:10 + hostname: kafka1 + user: "0" + ports: + - "9092:9092" + - "19092:19092" + - "29092:29092" + - "9999:9999" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://localhost:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CREATE_TOPICS: "akto.api.logs:3:3" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000 + KAFKA_LOG_RETENTION_HOURS: 5 + KAFKA_LOG_SEGMENT_BYTES: 104857600 + KAFKA_LOG_CLEANER_ENABLE: "true" + KAFKA_CLEANUP_POLICY: "delete" + KAFKA_LOG_RETENTION_BYTES: 10737418240 + volumes: + - ./data-kafka-data:/var/lib/kafka/data + - ./data-kafka-secrets:/etc/kafka/secrets + depends_on: + - zoo1 diff --git a/docker-compose-mongo.yml b/docker-compose-mongo.yml index 38dec35..a3aff1b 100644 --- a/docker-compose-mongo.yml +++ b/docker-compose-mongo.yml @@ -8,4 +8,4 @@ services: volumes: - ./data:/data/db ports: - - "27017:27017" \ No newline at end of file + - "27017:27017" diff --git a/docker-compose-threat-detection-backend.yml b/docker-compose-threat-detection-backend.yml new file mode 100644 index 0000000..f862408 --- /dev/null +++ b/docker-compose-threat-detection-backend.yml @@ -0,0 +1,53 @@ +version: '3.3' + +services: + mongo: + container_name: mongo + image: mongo + restart: on-failure:10 + volumes: + - ./data:/data/db + ports: + - "27017:27017" + + + kafka-internal: + image: confluentinc/cp-kafka:7.8.0 + container_name: kafka-internal + hostname: kafka-internal + user: root + ports: + - "29092:29092" # PLAINTEXT_HOST listener + - "19092:19092" # PLAINTEXT listener + - "9093:9093" # CONTROLLER listener + environment: + # Kafka Node Configuration + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,CONTROLLER://kafka-internal:9093,PLAINTEXT_HOST://0.0.0.0:29092 + + # Kafka Roles and Controller Quorum + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-internal:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + + # Other Configurations + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + + volumes: + - ./data-kafka-internal-data:/var/lib/kafka/data + - ./data-kafka-internal-secrets:/etc/kafka/secrets + + backend: + image: ag60/akto-threat-detection-backend + env_file: ./docker-threat-detection-backend.env + restart: always + ports: + - "8980:8980" + - "9090:9090" + depends_on: + - kafka-internal + - mongo \ No newline at end of file diff --git a/docker-compose-threat-detection-client.yml b/docker-compose-threat-detection-client.yml new file mode 100644 index 0000000..b00a9e9 --- /dev/null +++ b/docker-compose-threat-detection-client.yml @@ -0,0 +1,54 @@ +services: + postgres: + image: postgres:15 + container_name: postgres-internal + hostname: postgres-internal + ports: + - "5432:5432" + environment: + POSTGRES_USER: akto + POSTGRES_PASSWORD: akto + POSTGRES_DB: akto + volumes: + - postgres_data:/var/lib/postgresql/data + + kafka-internal: + image: confluentinc/cp-kafka:7.8.0 + container_name: kafka-internal + hostname: kafka-internal + user: root + ports: + - "29092:29092" # PLAINTEXT_HOST listener + - "19092:19092" # PLAINTEXT listener + - "9093:9093" # CONTROLLER listener + environment: + # Kafka Node Configuration + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-internal:19092,CONTROLLER://kafka-internal:9093,PLAINTEXT_HOST://0.0.0.0:29092 + + # Kafka Roles and Controller Quorum + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-internal:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + + # Other Configurations + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + + volumes: + - ./data-kafka-internal-data:/var/lib/kafka/data + - ./data-kafka-internal-secrets:/etc/kafka/secrets + + threat-detection: + image: ag60/akto-threat-detection + env_file: ./docker-threat-detection.env + restart: always + depends_on: + - kafka-internal + - postgres + +volumes: + postgres_data: \ No newline at end of file diff --git a/docker-compose-threat-detection-infra.yml b/docker-compose-threat-detection-infra.yml new file mode 100644 index 0000000..b2b12e5 --- /dev/null +++ b/docker-compose-threat-detection-infra.yml @@ -0,0 +1,55 @@ +services: + kafka-central: + image: confluentinc/cp-kafka:7.8.0 + container_name: kafka-central + hostname: kafka-central + user: root + ports: + - "9093:9093" # CONTROLLER listener + - "9094:9094" + environment: + # Broker ID + KAFKA_BROKER_ID: 1 + + # KRaft mode settings + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_NODE_ID: 1 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + + # Kafka listeners and advertised listeners + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,LISTENER_DOCKER_EXTERNAL_DIFFHOST://:9094 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092, LISTENER_DOCKER_EXTERNAL_DIFFHOST://10.0.143.202:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT + + # Specify controller listener names + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + + # Log directories + KAFKA_LOG_DIRS: /var/lib/kafka/data + + # Minimum in-sync replicas + KAFKA_MIN_INSYNC_REPLICAS: 1 + + # Offsets topic replication factor + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + + # Transaction state log replication factor and min ISR + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + + volumes: + - ./data-kafka-central-data:/var/lib/kafka/data + - ./data-kafka-central-secrets:/etc/kafka/secrets + + redis-central: + image: redis:latest + container_name: redis-central + hostname: redis-central + ports: + - "6379:6379" + volumes: + - redis_data:/data + +volumes: + redis_data: diff --git a/docker-compose.yml b/docker-compose.yml index 527aaed..b70bbf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://localhost:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" diff --git a/docker-threat-detection-backend.env b/docker-threat-detection-backend.env new file mode 100644 index 0000000..d1915c6 --- /dev/null +++ b/docker-threat-detection-backend.env @@ -0,0 +1,3 @@ +AKTO_MONGO_CONN=mongodb://mongo:27017 +AKTO_THREAT_PROTECTION_MONGO_CONN=mongodb://mongo:27017 +THREAT_EVENTS_KAFKA_BROKER_URL=kafka-internal:19092 diff --git a/docker-threat-detection.env b/docker-threat-detection.env new file mode 100644 index 0000000..ecb4866 --- /dev/null +++ b/docker-threat-detection.env @@ -0,0 +1,9 @@ +AKTO_MONGO_CONN= +AKTO_TRAFFIC_KAFKA_BOOTSTRAP_SERVER= +AKTO_INTERNAL_KAFKA_BOOTSTRAP_SERVER=kafka-internal:19092 +AKTO_THREAT_DETECTION_REDIS_URI=redis://redis1:6379 +AKTO_THREAT_DETECTION_POSTGRES=jdbc:postgresql://postgres-internal:5432/akto +AKTO_THREAT_DETECTION_POSTGRES_USER=akto +AKTO_THREAT_DETECTION_POSTGRES_PASSWORD=akto +AKTO_THREAT_PROTECTION_BACKEND_TOKEN= +AKTO_THREAT_PROTECTION_BACKEND_URL=https://tbs.akto.io \ No newline at end of file