Feature/segregation

cf-deploy-akto

@@ -105,16 +105,16 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then
     fi
   else
     log_step 'downloading...'
-    curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/self_hosting.zip
+    curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/segregation_2.zip
     log_step 'unpacking...'
     unzip master.zip
-    mv infra-feature-self_hosting infra
+    mv infra-feature-segregation_2 infra
   fi
 
   # NB: this is to make onprem containers to all get named the same.
   cd infra
   if ! command_present unzip; then
-    git checkout feature/self_hosting
+    git checkout feature/segregation_2
   fi
   DOCKER_CONTEXT="$(pwd)"
 
@@ -202,39 +202,39 @@ $MAYBE_SUDO docker-compose pull
 
 log_step "running Akto! ${WHITE}${BOLD}This can take up to 5 minutes${NORMAL}"
 
-export AKTO_KAFKA_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
-echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> .env
-$MAYBE_SUDO docker-compose up
+# export AKTO_KAFKA_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
+# echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> .env
+# $MAYBE_SUDO docker-compose up
 
-echo ""
-echo " -- ${GREEN}${BOLD}!! AKTO IS BOOTING !!${NORMAL} --"
+# echo ""
+# echo " -- ${GREEN}${BOLD}!! AKTO IS BOOTING !!${NORMAL} --"
 
-WAITED=0
+# WAITED=0
 
-sp="/-\|"
-echo -n ' '
-i=0
-while [ $WAITED -lt 100 ]; do
-  WAITED=$((WAITED + 1))
-  sleep 0.1
-  printf "\b${sp:i++%${#sp}:1}"
-done
+# sp="/-\|"
+# echo -n ' '
+# i=0
+# while [ $WAITED -lt 100 ]; do
+#   WAITED=$((WAITED + 1))
+#   sleep 0.1
+#   printf "\b${sp:i++%${#sp}:1}"
+# done
 
 
-echo " -- ${GREEN}${BOLD}!! AKTO LAUNCHED !!${NORMAL} --"
+# echo " -- ${GREEN}${BOLD}!! AKTO LAUNCHED !!${NORMAL} --"
 
 
-# NB: empty stuff to fully wipe out previous line completely
-printf "\r%s%s%s%s%s%s%s%s" "$MAGENTA" 'Check out your ' "$BOLD" 'BROWSER' "${NORMAL}${MAGENTA}" ' for some awesomeness!!!' "$NORMAL" '                              '
-echo
+# # NB: empty stuff to fully wipe out previous line completely
+# printf "\r%s%s%s%s%s%s%s%s" "$MAGENTA" 'Check out your ' "$BOLD" 'BROWSER' "${NORMAL}${MAGENTA}" ' for some awesomeness!!!' "$NORMAL" '                              '
+# echo
 
-echo ""
-echo "  ${CYAN}Navigate to${NORMAL}:          ${WHITE}${BOLD}http://localhost:8080/login${NORMAL}    or    ${WHITE}${BOLD}[publically_accessible_ip]:8080/login${NORMAL}"
-echo ""
+# echo ""
+# echo "  ${CYAN}Navigate to${NORMAL}:          ${WHITE}${BOLD}http://localhost:8080/login${NORMAL}    or    ${WHITE}${BOLD}[publically_accessible_ip]:8080/login${NORMAL}"
+# echo ""
 
-echo "Akto was installed in ~/akto. It will run in the background until you manually stop it. If Akto stops you can restart it without losing your data. "
+# echo "Akto was installed in ~/akto. It will run in the background until you manually stop it. If Akto stops you can restart it without losing your data. "
 
-if command_present open; then
-  open 'http://localhost:8080/login'
-fi
+# if command_present open; then
+#   open 'http://localhost:8080/login'
+# fi
 

cf-deploy-akto-start

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# NB: local trial script has to be self-contained
+# See https://sipb.mit.edu/doc/safe-shell/
+set -euf -o pipefail
+
+export MAYBE_SUDO=""
+pwd
+
+echo "starting....."
+
+if [ -t 1 ]; then
+  export NORMAL="$(tput sgr0)"
+  export RED="$(tput setaf 1)"
+  export GREEN="$(tput setaf 2)"
+  export MAGENTA="$(tput setaf 5)"
+  export CYAN="$(tput setaf 6)"
+  export WHITE="$(tput setaf 7)"
+  export BOLD="$(tput bold)"
+else
+  export NORMAL=""
+  export RED=""
+  export GREEN=""
+  export MAGENTA=""
+  export CYAN=""
+  export WHITE=""
+  export BOLD=""
+fi
+
+cd ~/akto/infra
+
+$MAYBE_SUDO docker-compose up
+
+echo ""
+echo " -- ${GREEN}${BOLD}!! AKTO IS BOOTING !!${NORMAL} --"
+
+WAITED=0
+
+sp="/-\|"
+echo -n ' '
+i=0
+while [ $WAITED -lt 100 ]; do
+  WAITED=$((WAITED + 1))
+  sleep 0.1
+  printf "\b${sp:i++%${#sp}:1}"
+done
+
+
+echo " -- ${GREEN}${BOLD}!! AKTO LAUNCHED !!${NORMAL} --"
+
+
+# NB: empty stuff to fully wipe out previous line completely
+printf "\r%s%s%s%s%s%s%s%s" "$MAGENTA" 'Check out your ' "$BOLD" 'BROWSER' "${NORMAL}${MAGENTA}" ' for some awesomeness!!!' "$NORMAL" '                              '
+echo
+
+echo ""
+echo "  ${CYAN}Navigate to${NORMAL}:          ${WHITE}${BOLD}http://localhost:8080/login${NORMAL}    or    ${WHITE}${BOLD}[publically_accessible_ip]:8080/login${NORMAL}"
+echo ""
+
+echo "Akto was installed in ~/akto. It will run in the background until you manually stop it. If Akto stops you can restart it without losing your data. "
+
+# if command_present open; then
+#   open 'http://localhost:8080/login'
+# fi
+

docker-compose-dashboard.yml

@@ -0,0 +1,9 @@
+version: '3.3'
+
+services:
+  akto-api-security-dashboard:
+    image: aktosecurity/akto-api-security-dashboard:latest
+    env_file: ./docker-dashboard.env
+    restart: always
+    ports: 
+      - "8080:8080"

docker-compose-mongo.yml

@@ -0,0 +1,11 @@
+version: '3.3'
+
+services:
+  mongo:
+    container_name: mongo
+    image: mongo
+    restart: on-failure:10
+    volumes:
+      - ./data:/data/db
+    ports:
+      - "27017:27017"

docker-compose-runtime.yml

@@ -0,0 +1,74 @@
+version: '2.1'
+
+services:
+  zoo1:
+    image: confluentinc/cp-zookeeper:6.2.1
+    restart: on-failure:10
+    hostname: zoo1
+    user: "0"  
+    volumes:
+      - ./data-zoo-data:/var/lib/zookeeper/data
+      - ./data-zoo-logs:/var/lib/zookeeper/log
+      - ./data-zoo-secrets:/etc/zookeeper/secrets              
+    container_name: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: zoo1:2888:3888
+
+  kafka1:
+    image: confluentinc/cp-kafka:6.2.1
+    restart: on-failure:10
+    hostname: kafka1
+    user: "0"
+    ports:
+      - "9092:9092"
+      - "19092:19092"
+      - "29092:29092"
+      - "9999:9999"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CREATE_TOPICS: "akto.api.logs:3:3"
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000
+      KAFKA_LOG_RETENTION_HOURS: 5
+      KAFKA_LOG_SEGMENT_BYTES: 104857600
+      KAFKA_LOG_CLEANER_ENABLE: "true"
+      KAFKA_CLEANUP_POLICY: "delete"
+      KAFKA_LOG_RETENTION_BYTES: 10737418240
+    volumes:
+      - ./data-kafka-data:/var/lib/kafka/data
+      - ./data-kafka-secrets:/etc/kafka/secrets             
+    depends_on:
+      - zoo1
+
+  akto-api-security-runtime:
+    image: aktosecurity/akto-api-security-runtime:latest
+    env_file: ./docker-runtime.env
+    mem_limit: 8g    
+    restart: always
+    depends_on:
+      - kafka1
+
+  akto-api-security-mirror-api-logging:
+    image: aktosecurity/mirror-api-logging:local
+    env_file: ./docker-runtime.env
+    mem_limit: 4g    
+    restart: always
+    depends_on:
+      - kafka1
+    network_mode: host
+
+  akto-hello-world:
+    image: hotavneesh/hello-world-server:latest
+    restart: always
+    ports: 
+      - "8000:8000"

docker-compose.yml

@@ -1,10 +1,15 @@
-version: '3.3'
+version: '2.1'
 
 services:
   zoo1:
     image: confluentinc/cp-zookeeper:6.2.1
     restart: on-failure:10
     hostname: zoo1
+    user: "0"  
+    volumes:
+      - ./data-zoo-data:/var/lib/zookeeper/data
+      - ./data-zoo-logs:/var/lib/zookeeper/log
+      - ./data-zoo-secrets:/etc/zookeeper/secrets          
     container_name: zoo1
     ports:
       - "2181:2181"
@@ -17,7 +22,7 @@ services:
     image: confluentinc/cp-kafka:6.2.1
     restart: on-failure:10
     hostname: kafka1
-    user: "appuser:appuser"
+    user: "0"
     ports:
       - "9092:9092"
       - "19092:19092"
@@ -33,6 +38,15 @@ services:
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_CREATE_TOPICS: "akto.api.logs:3:3"
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000
+      KAFKA_LOG_RETENTION_HOURS: 5
+      KAFKA_LOG_SEGMENT_BYTES: 104857600
+      KAFKA_LOG_CLEANER_ENABLE: "true"
+      KAFKA_CLEANUP_POLICY: "delete"
+      KAFKA_LOG_RETENTION_BYTES: 10737418240
+    volumes:
+      - ./data-kafka-data:/var/lib/kafka/data
+      - ./data-kafka-secrets:/etc/kafka/secrets       
     depends_on:
       - zoo1
 
@@ -49,6 +63,7 @@ services:
     image: aktosecurity/akto-api-security-dashboard:latest
     env_file: ./docker.env
     restart: always
+    mem_limit: 4g
     ports: 
       - "8080:8080"
     depends_on:
@@ -58,16 +73,16 @@ services:
     image: aktosecurity/akto-api-security-runtime:latest
     env_file: ./docker.env
     restart: always
+    mem_limit: 4g
     depends_on:
       - kafka1
       - mongo
 
   akto-api-security-mirror-api-logging:
-    image: aktosecurity/mirror-api-logging:local
+    image: aktosecurity/mirror-api-logging:latest
     env_file: ./docker.env
     restart: always
+    mem_limit: 4g
     depends_on:
       - kafka1
     network_mode: host
-
-

docker-dashboard.env

@@ -0,0 +1,10 @@
+# AKTO_MONGO_CONN=mongodb://mongo:27017/admini
+AKTO_CONFIG_NAME=staging
+AKTO_KAFKA_TOPIC_NAME=akto.api.logs
+# AKTO_KAFKA_BROKER_URL=kafka1:19092
+AKTO_KAFKA_BROKER_MAL=localhost:29092
+AKTO_KAFKA_GROUP_ID_CONFIG=asdf
+AKTO_KAFKA_MAX_POLL_RECORDS_CONFIG=100
+AKTO_ACCOUNT_NAME=Helios
+AKTO_TRAFFIC_BATCH_SIZE=100
+AKTO_TRAFFIC_BATCH_TIME_SECS=10

docker-runtime.env

@@ -0,0 +1,12 @@
+# AKTO_MONGO_CONN=mongodb://mongo:27017/admini
+AKTO_CONFIG_NAME=staging
+AKTO_KAFKA_TOPIC_NAME=akto.api.logs
+AKTO_KAFKA_BROKER_URL=kafka1:19092
+AKTO_KAFKA_BROKER_MAL=localhost:29092
+AKTO_KAFKA_GROUP_ID_CONFIG=asdf
+AKTO_KAFKA_MAX_POLL_RECORDS_CONFIG=100
+AKTO_ACCOUNT_NAME=Helios
+AKTO_TRAFFIC_BATCH_SIZE=100
+AKTO_TRAFFIC_BATCH_TIME_SECS=10
+USE_HOSTNAME=true
+