diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index 74cd11e11..b6dc86fbb 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -93,7 +93,7 @@ jobs: - name: Build image run: | - IMAGES=( $(grep image: docker-compose.yml | awk '{print $2}') ) + IMAGES=( $(grep image: docker-compose.yml | awk '{print $2}' | sort -u) ) for IMAGE in "${IMAGES[@]}"; do REPO_IMAGE="$(echo "$IMAGE" | sed "s@^\(malcolmnetsec\)@ghcr.io/${{ github.repository_owner }}/\1@" | sed "s/:.*/:${{ steps.extract_branch.outputs.branch }}/")" docker pull "$REPO_IMAGE" && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 15a57fc93..f00bf3c46 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -21,8 +21,8 @@ ENV PUSER_PRIV_DROP true ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm -ARG ZEEK_EXTRACTOR_PATH=/data/zeek/extract_files -ARG ZEEK_LOG_DIRECTORY=/data/zeek/logs +ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files +ARG ZEEK_LOG_DIRECTORY=/zeek/logs ARG EXTRACTED_FILE_IGNORE_EXISTING=false ARG EXTRACTED_FILE_PRESERVATION=quarantined ARG EXTRACTED_FILE_WATCHER_START_SLEEP=30 @@ -212,7 +212,7 @@ RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf USER root -WORKDIR /data/zeek/extract_files +WORKDIR /zeek/extract_files ENV PATH "${CAPA_DIR}:${PATH}" diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index bca1883b1..bb8e4d0cd 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -32,9 +32,10 @@ ARG FILEBEAT_CLOSE_RENAMED=true ARG FILEBEAT_CLOSE_REMOVED=true ARG FILEBEAT_CLOSE_EOF=true ARG FILEBEAT_CLEAN_REMOVED=true -ARG FILEBEAT_ZEEK_LOG_PATH="/data/zeek/current" -ARG FILEBEAT_SURICATA_LOG_PATH="/data/suricata" -ARG FILEBEAT_NGINX_LOG_PATH="/data/nginx" +ARG FILEBEAT_ZEEK_LOG_PATH="/zeek/current" +ARG FILEBEAT_ZEEK_LOG_LIVE_PATH="/zeek/live" +ARG FILEBEAT_SURICATA_LOG_PATH="/suricata" +ARG FILEBEAT_NGINX_LOG_PATH="/nginx" ARG LOG_CLEANUP_MINUTES=0 ARG ZIP_CLEANUP_MINUTES=0 ARG NGINX_LOG_ACCESS_AND_ERRORS=false @@ -77,8 +78,8 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml ADD filebeat/filebeat-tcp.yml /usr/share/filebeat-tcp/filebeat-tcp.yml -ADD filebeat/scripts /data/ -ADD shared/bin/opensearch_status.sh /data/ +ADD filebeat/scripts /usr/local/bin/ +ADD shared/bin/opensearch_status.sh /usr/local/bin/ ADD filebeat/supervisord.conf /etc/supervisord.conf RUN for INPUT in nginx tcp; do \ mkdir -p /usr/share/filebeat-$INPUT/data; \ @@ -87,8 +88,8 @@ RUN for INPUT in nginx tcp; do \ chmod 750 /usr/share/filebeat-$INPUT; \ chmod 770 /usr/share/filebeat-$INPUT/data; \ done; \ - chmod 755 /data/*.sh /data/*.py && \ - (echo -e "* * * * * /data/filebeat-process-zeek-folder.sh\n*/5 * * * * /data/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB}) + chmod 755 /usr/local/bin/*.sh /usr/local/bin/*.py && \ + (echo -e "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB}) ENV AUTO_TAG $AUTO_TAG ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES @@ -102,6 +103,7 @@ ENV FILEBEAT_CLOSE_REMOVED $FILEBEAT_CLOSE_REMOVED ENV FILEBEAT_CLOSE_EOF $FILEBEAT_CLOSE_EOF ENV FILEBEAT_CLEAN_REMOVED $FILEBEAT_CLEAN_REMOVED ENV FILEBEAT_ZEEK_LOG_PATH $FILEBEAT_ZEEK_LOG_PATH +ENV FILEBEAT_ZEEK_LOG_LIVE_PATH $FILEBEAT_ZEEK_LOG_LIVE_PATH ENV FILEBEAT_SURICATA_LOG_PATH $FILEBEAT_SURICATA_LOG_PATH ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS @@ -120,8 +122,7 @@ ENV FILEBEAT_TCP_PARSE_TARGET_FIELD $FILEBEAT_TCP_PARSE_TARGET_FIELD ENV FILEBEAT_TCP_PARSE_DROP_FIELD $FILEBEAT_TCP_PARSE_DROP_FIELD ENV FILEBEAT_TCP_TAG $FILEBEAT_TCP_TAG ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json" -ENV FILEBEAT_ZEEK_DIR "/data/zeek/" -ENV PATH="/data:${PATH}" +ENV FILEBEAT_ZEEK_DIR "/zeek/" VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-tcp/data"] diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index 02e3c558a..ede844667 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -30,7 +30,7 @@ ENV TERM xterm ARG PCAP_ENABLE_TCPDUMP=false ARG PCAP_ENABLE_NETSNIFF=false # PCAP_IFACE=comma-separated list of capture interfaces -ARG PCAP_IFACE=eth0 +ARG PCAP_IFACE=lo ARG PCAP_NETSNIFF_MAGIC=0xa1b2c3d4 ARG PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d%H%M%S.pcap ARG PCAP_ROTATE_MINUTES=30 diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 6d8dd2848..376a06514 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -101,7 +101,13 @@ ENV DEFAULT_UID $DEFAULT_UID ENV DEFAULT_GID $DEFAULT_GID ENV PUSER "suricata" ENV PGROUP "suricata" -ENV PUSER_PRIV_DROP true +# not dropping privileges globally: supervisord will take care of it +# for all processes, but first we need root to sure capabilities for +# traffic capturing tools are in-place before they are started. +# despite doing setcap here in the Dockerfile, the chown in +# docker-uid-gid-setup.sh will cause them to be lost, so we need +# a final check in docker_entrypoint.sh before startup +ENV PUSER_PRIV_DROP false ENV SUPERCRONIC_VERSION "0.2.1" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" @@ -125,12 +131,16 @@ COPY --from=builder /suricatabld.tar.gz /suricatabld.tar.gz RUN apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get install -q -y --no-install-recommends \ + bc \ curl \ + ethtool \ file \ inotify-tools \ + iproute2 \ jq \ less \ libcap-ng0 \ + libcap2-bin \ libevent-2.1-7 \ libevent-pthreads-2.1-7 \ libgeoip1 \ @@ -175,6 +185,9 @@ RUN apt-get -q update && \ ln -sfr /opt/pcap_processor.py /opt/pcap_suricata_processor.py && \ (echo "*/5 * * * * /opt/eve-clean-logs.sh" > ${SUPERCRONIC_CRONTAB}) && \ tar xf /suricatabld.tar.gz --strip-components=1 -C / && \ + chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \ mkdir -p "$SURICATA_CUSTOM_RULES_DIR" && \ chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" && \ apt-get clean && \ @@ -193,19 +206,32 @@ ARG PCAP_PIPELINE_DEBUG=false ARG PCAP_PIPELINE_DEBUG_EXTRA=false ARG PCAP_MONITOR_HOST=pcap-monitor ARG AUTO_TAG=true +ARG SURICATA_PCAP_PROCESSOR=true +ARG SURICATA_CRON=true ARG SURICATA_AUTO_ANALYZE_PCAP_FILES=false ARG SURICATA_CUSTOM_RULES_ONLY=false ARG SURICATA_AUTO_ANALYZE_PCAP_THREADS=1 ARG LOG_CLEANUP_MINUTES=30 +ARG SURICATA_LIVE_CAPTURE=false +ARG SURICATA_ROTATED_PCAP=false +# PCAP_IFACE=comma-separated list of capture interfaces +ARG PCAP_IFACE=lo +ARG PCAP_FILTER= ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG -ENV AUTO_TAG $AUTO_TAG ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST +ENV AUTO_TAG $AUTO_TAG +ENV SURICATA_PCAP_PROCESSOR $SURICATA_PCAP_PROCESSOR +ENV SURICATA_CRON $SURICATA_CRON ENV SURICATA_AUTO_ANALYZE_PCAP_FILES $SURICATA_AUTO_ANALYZE_PCAP_FILES ENV SURICATA_AUTO_ANALYZE_PCAP_THREADS $SURICATA_AUTO_ANALYZE_PCAP_THREADS ENV SURICATA_CUSTOM_RULES_ONLY $SURICATA_CUSTOM_RULES_ONLY ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES +ENV SURICATA_LIVE_CAPTURE $SURICATA_LIVE_CAPTURE +ENV SURICATA_ROTATED_PCAP $SURICATA_ROTATED_PCAP +ENV PCAP_IFACE $PCAP_IFACE +ENV PCAP_FILTER $PCAP_FILTER ENV PUSER_CHOWN "$SURICATA_CONFIG_DIR;$SURICATA_MANAGED_DIR;$SURICATA_LOG_DIR;$SURICATA_RUN_DIR" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index aee61f6f5..8a91e12f5 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -21,7 +21,13 @@ ENV DEFAULT_UID $DEFAULT_UID ENV DEFAULT_GID $DEFAULT_GID ENV PUSER "zeeker" ENV PGROUP "zeeker" -ENV PUSER_PRIV_DROP true +# not dropping privileges globally: supervisord will take care of it +# for all processes, but first we need root to sure capabilities for +# traffic capturing tools are in-place before they are started. +# despite doing setcap here in the Dockerfile, the chown in +# docker-uid-gid-setup.sh will cause them to be lost, so we need +# a final check in docker_entrypoint.sh before startup +ENV PUSER_PRIV_DROP false # for download and install ARG ZEEK_LTS= @@ -48,20 +54,24 @@ ENV PATH "${ZEEK_DIR}/bin:${PATH}" ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/ # build and install system packages, zeek, spicy and plugins -RUN apt-get -q update && \ +RUN export DEBARCH=$(dpkg --print-architecture) && \ + apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get install -q -y --no-install-recommends \ + bc \ bison \ ca-certificates \ ccache \ cmake \ curl \ + ethtool \ file \ flex \ g++ \ gcc \ git \ gnupg2 \ + iproute2 \ jq \ less \ libatomic1 \ @@ -77,6 +87,7 @@ RUN apt-get -q update && \ libtcmalloc-minimal4 \ libunwind8 \ libzmq5 \ + linux-headers-$DEBARCH \ locales-all \ make \ moreutils \ @@ -145,7 +156,9 @@ ADD shared/pcaps /tmp/pcaps ADD zeek/supervisord.conf /etc/supervisord.conf ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/ ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/ -ADD shared/bin/zeek_intel_setup.sh /usr/local/bin/entrypoint.sh +ADD zeek/scripts/docker_entrypoint.sh /usr/local/bin/ +ADD shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/ +ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh @@ -166,12 +179,20 @@ RUN mkdir -p /tmp/logs && \ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ usermod -a -G tty ${PUSER} && \ + chown root:${PGROUP} /sbin/ethtool "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats && \ touch "${SUPERCRONIC_CRONTAB}" && \ chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel "${SUPERCRONIC_CRONTAB}" && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_zeek_processor.py #Whether or not to auto-tag logs based on filename ARG AUTO_TAG=true +#Whether or not to start up the pcap_processor script to monitor pcaps +ARG ZEEK_PCAP_PROCESSOR=true +#Whether or not to start up supercronic for updating intel definitions +ARG ZEEK_CRON=true #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1 @@ -184,8 +205,15 @@ ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files ARG PCAP_PIPELINE_DEBUG=false ARG PCAP_PIPELINE_DEBUG_EXTRA=false ARG PCAP_MONITOR_HOST=pcap-monitor +ARG ZEEK_LIVE_CAPTURE=false +ARG ZEEK_ROTATED_PCAP=false +# PCAP_IFACE=comma-separated list of capture interfaces +ARG PCAP_IFACE=lo +ARG PCAP_FILTER= ENV AUTO_TAG $AUTO_TAG +ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR +ENV ZEEK_CRON $ZEEK_CRON ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION @@ -197,6 +225,10 @@ ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST +ENV ZEEK_LIVE_CAPTURE $ZEEK_LIVE_CAPTURE +ENV ZEEK_ROTATED_PCAP $ZEEK_ROTATED_PCAP +ENV PCAP_IFACE $PCAP_IFACE +ENV PCAP_FILTER $PCAP_FILTER # environment variables for zeek runtime tweaks (used in local.zeek) ARG ZEEK_DISABLE_HASH_ALL_FILES= @@ -234,9 +266,11 @@ ENV ZEEK_DISABLE_SPICY_TAILSCALE $ZEEK_DISABLE_SPICY_TAILSCALE ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD +ENV PUSER_CHOWN "$ZEEK_DIR" + VOLUME ["${ZEEK_DIR}/share/zeek/site/intel"] -ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/entrypoint.sh"] +ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker_entrypoint.sh"] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"] diff --git a/README.md b/README.md index cecadd68e..e7cfaeb15 100644 --- a/README.md +++ b/README.md @@ -183,23 +183,23 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 6.1.1 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 6.1.1 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 6.1.1 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 6.1.1 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/filebeat-oss 6.1.1 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/file-monitor 6.1.1 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 6.1.1 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/freq 6.1.1 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 6.1.1 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 6.1.1 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 6.1.1 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/nginx-proxy 6.1.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 6.1.1 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 6.1.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 6.1.1 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/suricata 6.1.1 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 6.1.1 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 6.2.0 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 6.2.0 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 6.2.0 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 6.2.0 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/filebeat-oss 6.2.0 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/file-monitor 6.2.0 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 6.2.0 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/freq 6.2.0 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 6.2.0 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 6.2.0 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 6.2.0 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/nginx-proxy 6.2.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 6.2.0 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 6.2.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 6.2.0 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/suricata 6.2.0 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 6.2.0 xxxxxxxxxxxx 3 days ago 1GB ``` #### Import from pre-packaged tarballs @@ -561,8 +561,8 @@ Various other environment variables inside of `docker-compose.yml` can be tweake * `PCAP_ENABLE_TCPDUMP` – if set to `true`, Malcolm will capture network traffic on the local network interface(s) indicated in `PCAP_IFACE` using [tcpdump](https://www.tcpdump.org/); there is no reason to enable *both* `PCAP_ENABLE_NETSNIFF` and `PCAP_ENABLE_TCPDUMP` * `PCAP_FILTER` – specifies a tcpdump-style filter expression for local packet capture; leave blank to capture all traffic * `PCAP_IFACE` – used to specify the network interface(s) for local packet capture if `PCAP_ENABLE_NETSNIFF` or `PCAP_ENABLE_TCPDUMP` are enabled; for multiple interfaces, separate the interface names with a comma (e.g., `'enp0s25'` or `'enp10s0,enp11s0'`) -* `PCAP_ROTATE_MEGABYTES` – used to specify how large a locally-captured PCAP file can become (in megabytes) before it closed for processing and a new PCAP file created -* `PCAP_ROTATE_MINUTES` – used to specify an time interval (in minutes) after which a locally-captured PCAP file will be closed for processing and a new PCAP file created +* `PCAP_ROTATE_MEGABYTES` – used to specify how large a locally-captured PCAP file can become (in megabytes) before it is closed for processing and a new PCAP file created +* `PCAP_ROTATE_MINUTES` – used to specify a time interval (in minutes) after which a locally-captured PCAP file will be closed for processing and a new PCAP file created * `pipeline.workers`, `pipeline.batch.size` and `pipeline.batch.delay` - these settings are used to tune the performance and resource utilization of the the `logstash` container; see [Tuning and Profiling Logstash Performance](https://www.elastic.co/guide/en/logstash/current/tuning-logstash.html), [`logstash.yml`](https://www.elastic.co/guide/en/logstash/current/logstash-settings-file.html) and [Multiple Pipelines](https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html) * `PUID` and `PGID` - Docker runs all of its containers as the privileged `root` user by default. For better security, Malcolm immediately drops to non-privileged user accounts for executing internal processes wherever possible. The `PUID` (**p**rocess **u**ser **ID**) and `PGID` (**p**rocess **g**roup **ID**) environment variables allow Malcolm to map internal non-privileged user accounts to a corresponding [user account](https://en.wikipedia.org/wiki/User_identifier) on the host. * `QUESTIONABLE_COUNTRY_CODES` - when [severity scoring](#Severity) is enabled, this variable defines a comma-separated list of countries of concern (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'CN,IR,KP,RU,UA'`) @@ -1420,7 +1420,7 @@ Restarting Logstash may take several minutes, after which log ingestion will be ### OpenSearch index management -Malcolm releases prior to v6.1.1 used environment variables to configure OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) [policies](https://opensearch.org/docs/latest/im-plugin/ism/policies/). +Malcolm releases prior to v6.2.0 used environment variables to configure OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) [policies](https://opensearch.org/docs/latest/im-plugin/ism/policies/). Since then, OpenSearch Dashboards has developed and released plugins with UIs for [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/). Because these plugins provide a more comprehensive and user-friendly interfaces for these features, the old environment variable-based configuration code has been removed from Malcolm, with the exception of the code that uses `OPENSEARCH_INDEX_SIZE_PRUNE_LIMIT` and `OPENSEARCH_INDEX_SIZE_PRUNE_NAME_SORT` which deals with deleting the oldest network session metadata indices when the database exceeds a certain size. @@ -3425,7 +3425,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-6.1.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-6.2.0.iso" … ``` @@ -3835,23 +3835,23 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 6.1.1 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 6.1.1 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 6.1.1 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 6.1.1 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/filebeat-oss 6.1.1 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/file-monitor 6.1.1 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 6.1.1 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/freq 6.1.1 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 6.1.1 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 6.1.1 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 6.1.1 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/nginx-proxy 6.1.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 6.1.1 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 6.1.1 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 6.1.1 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/suricata 6.1.1 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 6.1.1 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 6.2.0 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 6.2.0 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 6.2.0 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 6.2.0 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/filebeat-oss 6.2.0 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/file-monitor 6.2.0 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 6.2.0 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/freq 6.2.0 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 6.2.0 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 6.2.0 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 6.2.0 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/nginx-proxy 6.2.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 6.2.0 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 6.2.0 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 6.2.0 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/suricata 6.2.0 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 6.2.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index c97754509..7d5a4fcc3 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -164,7 +164,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: opensearch: - image: malcolmnetsec/opensearch:6.1.1 + image: malcolmnetsec/opensearch:6.2.0 restart: "no" stdin_open: false tty: true @@ -203,7 +203,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: malcolmnetsec/dashboards-helper:6.1.1 + image: malcolmnetsec/dashboards-helper:6.2.0 restart: "no" stdin_open: false tty: true @@ -231,7 +231,7 @@ services: retries: 3 start_period: 30s dashboards: - image: malcolmnetsec/dashboards:6.1.1 + image: malcolmnetsec/dashboards:6.2.0 restart: "no" stdin_open: false tty: true @@ -254,7 +254,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:6.1.1 + image: malcolmnetsec/logstash-oss:6.2.0 restart: "no" stdin_open: false tty: true @@ -299,7 +299,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:6.1.1 + image: malcolmnetsec/filebeat-oss:6.2.0 restart: "no" stdin_open: false tty: true @@ -313,6 +313,7 @@ services: << : *common-upload-variables << : *common-beats-variables FILEBEAT_ZEEK_LOG_PATH : '/data/zeek/current' + FILEBEAT_ZEEK_LOG_LIVE_PATH : '/data/zeek/live' FILEBEAT_SURICATA_LOG_PATH : '/data/suricata' FILEBEAT_NGINX_LOG_PATH : '/data/nginx' depends_on: @@ -335,7 +336,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:6.1.1 + image: malcolmnetsec/arkime:6.2.0 restart: "no" stdin_open: false tty: true @@ -375,7 +376,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:6.1.1 + image: malcolmnetsec/zeek:6.2.0 restart: "no" stdin_open: false tty: true @@ -406,7 +407,7 @@ services: retries: 3 start_period: 60s suricata: - image: malcolmnetsec/suricata:6.1.1 + image: malcolmnetsec/suricata:6.2.0 restart: "no" stdin_open: false tty: true @@ -431,7 +432,7 @@ services: retries: 3 start_period: 120s file-monitor: - image: malcolmnetsec/file-monitor:6.1.1 + image: malcolmnetsec/file-monitor:6.2.0 restart: "no" stdin_open: false tty: true @@ -456,7 +457,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:6.1.1 + image: malcolmnetsec/pcap-capture:6.2.0 restart: "no" stdin_open: false tty: true @@ -476,7 +477,7 @@ services: volumes: - ./pcap/upload:/pcap pcap-monitor: - image: malcolmnetsec/pcap-monitor:6.1.1 + image: malcolmnetsec/pcap-monitor:6.2.0 restart: "no" stdin_open: false tty: true @@ -501,7 +502,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:6.1.1 + image: malcolmnetsec/file-upload:6.2.0 restart: "no" stdin_open: false tty: true @@ -529,7 +530,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:6.1.1 + image: malcolmnetsec/htadmin:6.2.0 restart: "no" stdin_open: false tty: true @@ -553,7 +554,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:6.1.1 + image: malcolmnetsec/freq:6.2.0 restart: "no" stdin_open: false tty: true @@ -573,7 +574,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:6.1.1 + image: malcolmnetsec/name-map-ui:6.2.0 restart: "no" stdin_open: false tty: true @@ -596,7 +597,7 @@ services: retries: 3 start_period: 60s api: - image: malcolmnetsec/api:6.1.1 + image: malcolmnetsec/api:6.2.0 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -616,7 +617,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:6.1.1 + image: malcolmnetsec/nginx-proxy:6.2.0 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index da07e0ab2..46b878cbb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,7 +167,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: malcolmnetsec/opensearch:6.1.1 + image: malcolmnetsec/opensearch:6.2.0 restart: "no" stdin_open: false tty: true @@ -209,7 +209,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: malcolmnetsec/dashboards-helper:6.1.1 + image: malcolmnetsec/dashboards-helper:6.2.0 restart: "no" stdin_open: false tty: true @@ -240,7 +240,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: malcolmnetsec/dashboards:6.1.1 + image: malcolmnetsec/dashboards:6.2.0 restart: "no" stdin_open: false tty: true @@ -266,7 +266,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:6.1.1 + image: malcolmnetsec/logstash-oss:6.2.0 restart: "no" stdin_open: false tty: true @@ -318,7 +318,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:6.1.1 + image: malcolmnetsec/filebeat-oss:6.2.0 restart: "no" stdin_open: false tty: true @@ -332,6 +332,7 @@ services: << : *common-upload-variables << : *common-beats-variables FILEBEAT_ZEEK_LOG_PATH : '/data/zeek/current' + FILEBEAT_ZEEK_LOG_LIVE_PATH : '/data/zeek/live' FILEBEAT_SURICATA_LOG_PATH : '/data/suricata' FILEBEAT_NGINX_LOG_PATH : '/data/nginx' depends_on: @@ -357,7 +358,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:6.1.1 + image: malcolmnetsec/arkime:6.2.0 restart: "no" stdin_open: false tty: true @@ -403,7 +404,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:6.1.1 + image: malcolmnetsec/zeek:6.2.0 restart: "no" stdin_open: false tty: true @@ -438,7 +439,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:6.1.1 + image: malcolmnetsec/suricata:6.2.0 restart: "no" stdin_open: false tty: true @@ -466,7 +467,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:6.1.1 + image: malcolmnetsec/file-monitor:6.2.0 restart: "no" stdin_open: false tty: true @@ -494,7 +495,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:6.1.1 + image: malcolmnetsec/pcap-capture:6.2.0 restart: "no" stdin_open: false tty: true @@ -517,7 +518,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:6.1.1 + image: malcolmnetsec/pcap-monitor:6.2.0 restart: "no" stdin_open: false tty: true @@ -545,7 +546,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:6.1.1 + image: malcolmnetsec/file-upload:6.2.0 restart: "no" stdin_open: false tty: true @@ -573,7 +574,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:6.1.1 + image: malcolmnetsec/htadmin:6.2.0 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -600,7 +601,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:6.1.1 + image: malcolmnetsec/freq:6.2.0 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -623,7 +624,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:6.1.1 + image: malcolmnetsec/name-map-ui:6.2.0 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -649,7 +650,7 @@ services: retries: 3 start_period: 60s api: - image: malcolmnetsec/api:6.1.1 + image: malcolmnetsec/api:6.2.0 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -675,7 +676,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:6.1.1 + image: malcolmnetsec/nginx-proxy:6.2.0 restart: "no" stdin_open: false tty: true diff --git a/docs/web/download.md b/docs/web/download.md index fab386fcc..374398c59 100644 --- a/docs/web/download.md +++ b/docs/web/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-6.1.1.iso](/iso/malcolm-6.1.1.iso) (4.5GiB) | [`xxxxxxxx`](/iso/malcolm-6.1.1.iso.sha256.txt) | +| [malcolm-6.2.0.iso](/iso/malcolm-6.2.0.iso) (4.5GiB) | [`xxxxxxxx`](/iso/malcolm-6.2.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-6.1.1.iso](/iso/hedgehog-6.1.1.iso) (2.1GiB) | [`xxxxxxxx`](/iso/hedgehog-6.1.1.iso.sha256.txt) | +| [hedgehog-6.2.0.iso](/iso/hedgehog-6.2.0.iso) (2.1GiB) | [`xxxxxxxx`](/iso/hedgehog-6.2.0.iso.sha256.txt) | ## Warning diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 9130c500e..6a0bb3ed1 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -30,7 +30,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -50,7 +50,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -68,7 +68,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -86,7 +86,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -104,7 +104,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -122,7 +122,7 @@ startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true @@ -157,14 +157,14 @@ redirect_stderr=true command=/usr/local/bin/zeek_carved_http_server.py --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s - --directory /data/zeek/extract_files + --directory /zeek/extract_files autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s autorestart=true startsecs=0 startretries=0 stopasgroup=true killasgroup=true -directory=/data/zeek/extract_files +directory=/zeek/extract_files stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true diff --git a/filebeat/filebeat-nginx.yml b/filebeat/filebeat-nginx.yml index 3f439080e..b588c68aa 100644 --- a/filebeat/filebeat-nginx.yml +++ b/filebeat/filebeat-nginx.yml @@ -11,10 +11,10 @@ filebeat.modules: - module: nginx access: enabled: true - var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/data/nginx}/access.log*"] + var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/nginx}/access.log*"] error: enabled: true - var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/data/nginx}/error.log*"] + var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/nginx}/error.log*"] processors: - add_tags: diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index f45bc0aa9..ad47211ce 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -6,11 +6,11 @@ logging.metrics.enabled: false #================================ Inputs ======================================= -#-------------------------- Current "real" Zeek logs --------------------------- +#-------------------------- "Real" Zeek logs ----------------------------------- filebeat.inputs: - type: log paths: - - ${FILEBEAT_ZEEK_LOG_PATH:/data/zeek/current}/*.log + - ${FILEBEAT_ZEEK_LOG_PATH:/zeek/current}/*.log # see comment below for signatures(_carved).log exclude_files: ['signatures\(_carved.*\)\.log$'] symlinks: true @@ -19,14 +19,31 @@ filebeat.inputs: compression_level: 0 exclude_lines: ['^\s*#'] scan_frequency: ${FILEBEAT_SCAN_FREQUENCY:10s} - clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:45m} - ignore_older: ${FILEBEAT_IGNORE_OLDER:30m} - close_inactive: ${FILEBEAT_CLOSE_INACTIVE:30s} + clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:180m} + ignore_older: ${FILEBEAT_IGNORE_OLDER:120m} + close_inactive: ${FILEBEAT_CLOSE_INACTIVE:120s} close_renamed: ${FILEBEAT_CLOSE_RENAMED:true} close_removed: ${FILEBEAT_CLOSE_REMOVED:true} close_eof: ${FILEBEAT_CLOSE_EOF:true} clean_removed: ${FILEBEAT_CLEAN_REMOVED:true} +- type: log + paths: + - ${FILEBEAT_ZEEK_LOG_LIVE_PATH:/zeek/live}/logs/current/*.log + symlinks: true + fields_under_root: true + tags: ["_filebeat_zeek_live"] + compression_level: 0 + exclude_lines: ['^\s*#'] + scan_frequency: ${FILEBEAT_SCAN_FREQUENCY:10s} + clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:180m} + ignore_older: ${FILEBEAT_IGNORE_OLDER:120m} + close_inactive: ${FILEBEAT_CLOSE_INACTIVE_LIVE:90m} + close_renamed: ${FILEBEAT_CLOSE_RENAMED:true} + close_removed: ${FILEBEAT_CLOSE_REMOVED:true} + close_eof: false + clean_removed: ${FILEBEAT_CLEAN_REMOVED:true} + #-------------------------- Carved file signature hit log ---------------------- # signatures(_carved).log is different, as it comes from file carving and is # "live" regardless of whether the other *.log files that may be processed @@ -37,7 +54,7 @@ filebeat.inputs: # custom settings here. - type: log paths: - - ${FILEBEAT_ZEEK_LOG_PATH:/data/zeek/current}/signatures(_carved*).log + - ${FILEBEAT_ZEEK_LOG_PATH:/zeek/current}/signatures(_carved*).log symlinks: true fields_under_root: true tags: ["_filebeat_zeek"] @@ -55,20 +72,36 @@ filebeat.inputs: #-------------------------- Suricata EVE JSON logs ----------------------------- - type: log paths: - - ${FILEBEAT_SURICATA_LOG_PATH:/data/suricata}/eve-*.json + - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/eve-*.json symlinks: true fields_under_root: true tags: ["_filebeat_suricata"] compression_level: 0 scan_frequency: ${FILEBEAT_SCAN_FREQUENCY:10s} - clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:45m} - ignore_older: ${FILEBEAT_IGNORE_OLDER:30m} - close_inactive: ${FILEBEAT_CLOSE_INACTIVE:30s} + clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:180m} + ignore_older: ${FILEBEAT_IGNORE_OLDER:120m} + close_inactive: ${FILEBEAT_CLOSE_INACTIVE:120s} close_renamed: ${FILEBEAT_CLOSE_RENAMED:true} close_removed: ${FILEBEAT_CLOSE_REMOVED:true} close_eof: ${FILEBEAT_CLOSE_EOF:true} clean_removed: ${FILEBEAT_CLEAN_REMOVED:true} +- type: log + paths: + - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/live/eve.json + symlinks: true + fields_under_root: true + tags: ["_filebeat_suricata_live"] + compression_level: 0 + scan_frequency: ${FILEBEAT_SCAN_FREQUENCY:10s} + clean_inactive: ${FILEBEAT_CLEAN_INACTIVE:180m} + ignore_older: ${FILEBEAT_IGNORE_OLDER:120m} + close_inactive: ${FILEBEAT_CLOSE_INACTIVE_LIVE:90m} + close_renamed: ${FILEBEAT_CLOSE_RENAMED:true} + close_removed: ${FILEBEAT_CLOSE_REMOVED:true} + close_eof: false + clean_removed: ${FILEBEAT_CLEAN_REMOVED:true} + #================================ Outputs ====================================== #-------------------------- Logstash Output ------------------------------------ diff --git a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py index 0bc60c21f..cdd4a1ff4 100755 --- a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py +++ b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py @@ -17,14 +17,19 @@ from subprocess import Popen, PIPE lockFilename = os.path.join(gettempdir(), '{}.lock'.format(os.path.basename(__file__))) -broDir = os.path.join(os.getenv('FILEBEAT_ZEEK_DIR', "/data/zeek/"), '') +zeekDir = os.path.join(os.getenv('FILEBEAT_ZEEK_DIR', "/zeek/"), '') cleanLogSeconds = int(os.getenv('LOG_CLEANUP_MINUTES', "30")) * 60 cleanZipSeconds = int(os.getenv('ZIP_CLEANUP_MINUTES', "120")) * 60 fbRegFilename = os.getenv('FILEBEAT_REGISTRY_FILE', "/usr/share/filebeat/data/registry/filebeat/data.json") -currentDir = broDir + "current/" -processedDir = broDir + "processed/" +currentDir = zeekDir + "current/" +processedDir = zeekDir + "processed/" +liveDir = zeekDir + "live/logs/" -import os, errno +nowTime = time.time() +logMimeType = "text/plain" +archiveMimeTypeRegex = re.compile( + r"(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)" +) def silentRemove(filename): @@ -37,36 +42,12 @@ def silentRemove(filename): pass -def pruneFiles(): - - if (cleanLogSeconds <= 0) and (cleanZipSeconds <= 0): - # disabled, don't do anything - return - - nowTime = time.time() - - logMimeType = "text/plain" - archiveMimeTypeRegex = re.compile( - r"(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)" - ) - - # look for regular files in the processed/ directory - foundFiles = [ - (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(processedDir) for filename in filenames - ] - - # look up the filebeat registry file and try to read it - fbReg = None - if os.path.isfile(fbRegFilename): - with open(fbRegFilename) as f: - fbReg = json.load(f) - - # see if the files we found are in use and old enough to be pruned - for file in foundFiles: +def checkFile(filename, filebeatReg=None, checkLogs=True, checkArchives=True): + try: # first check to see if it's in the filebeat registry - if fbReg is not None: - fileStatInfo = os.stat(file) + if filebeatReg is not None: + fileStatInfo = os.stat(filename) if fileStatInfo: fileFound = any( ( @@ -74,31 +55,31 @@ def pruneFiles(): and (entry['FileStateOS']['device'] == fileStatInfo.st_dev) and (entry['FileStateOS']['inode'] == fileStatInfo.st_ino) ) - for entry in fbReg + for entry in filebeatReg ) if fileFound: # found a file in the filebeat registry, so leave it alone! # we only want to delete files that filebeat has forgotten - # print "{} is found in registry!".format(file) - continue + # print "{} is found in registry!".format(filename) + return # else: - # print "{} is NOT found in registry!".format(file) + # print "{} is NOT found in registry!".format(filename) # now see if the file is in use by any other process in the system - fuserProcess = Popen(["fuser", "-s", file], stdout=PIPE) + fuserProcess = Popen(["fuser", "-s", filename], stdout=PIPE) fuserProcess.communicate() fuserExitCode = fuserProcess.wait() if fuserExitCode != 0: # the file is not in use, let's check it's mtime/ctime - logTime = max(os.path.getctime(file), os.path.getmtime(file)) + logTime = max(os.path.getctime(filename), os.path.getmtime(filename)) lastUseTime = nowTime - logTime # get the file type - fileType = magic.from_file(file, mime=True) - if (cleanLogSeconds > 0) and (fileType == logMimeType): + fileType = magic.from_file(filename, mime=True) + if (checkLogs == True) and (cleanLogSeconds > 0) and (fileType == logMimeType): cleanSeconds = cleanLogSeconds - elif (cleanZipSeconds > 0) and archiveMimeTypeRegex.match(fileType) is not None: + elif (checkArchives == True) and (cleanZipSeconds > 0) and archiveMimeTypeRegex.match(fileType) is not None: cleanSeconds = cleanZipSeconds else: # not a file we're going to be messing with @@ -106,8 +87,44 @@ def pruneFiles(): if (cleanSeconds > 0) and (lastUseTime >= cleanSeconds): # this is a closed file that is old, so delete it - print('removing old file "{}" ({}, used {} seconds ago)'.format(file, fileType, lastUseTime)) - silentRemove(file) + print('removing old file "{}" ({}, used {} seconds ago)'.format(filename, fileType, lastUseTime)) + silentRemove(filename) + + except FileNotFoundError as fnf: + # file's already gone, oh well + pass + + except Exception as e: + print("{} for '{}': {}".format(type(e).__name__, filename, e)) + + +def pruneFiles(): + + if (cleanLogSeconds <= 0) and (cleanZipSeconds <= 0): + # disabled, don't do anything + return + + # look for regular files in the processed/ directory + foundFiles = [ + (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(processedDir) for filename in filenames + ] + + # look for rotated files from live zeek instance + rotatedFiles = [ + (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(liveDir) for filename in filenames + ] + + # look up the filebeat registry file and try to read it + fbReg = None + if os.path.isfile(fbRegFilename): + with open(fbRegFilename) as f: + fbReg = json.load(f) + + # see if the files we found are in use and old enough to be pruned + for file in foundFiles: + checkFile(file, filebeatReg=fbReg, checkLogs=True, checkArchives=True) + for file in rotatedFiles: + checkFile(file, filebeatReg=None, checkLogs=False, checkArchives=True) # clean up any broken symlinks in the current/ directory for current in os.listdir(currentDir): diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 63ec0a8ee..d809cca2b 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -3,8 +3,8 @@ # Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved. -# for files (sort -V (natural)) under /data/zeek that: -# - are not in processed/ or current/ or upload/ or extract_files/ (-prune) +# for files (sort -V (natural)) under /zeek that: +# - are not in processed/ or current/ or upload/ or extract_files/ or live/ (-prune) # - are archive files # - are not in use (fuser -s) # 1. move file to processed/ (preserving original subdirectory heirarchy, if any) @@ -22,7 +22,7 @@ export ZEEK_LOG_FIELD_BITMAP_SCRIPT="$SCRIPT_DIR/zeek-log-field-bitmap.py" export ZEEK_LOG_AUTO_TAG=${AUTO_TAG:-"true"} -ZEEK_LOGS_DIR=${FILEBEAT_ZEEK_DIR:-/data/zeek/} +ZEEK_LOGS_DIR=${FILEBEAT_ZEEK_DIR:-/zeek/} # remove the lock directory on exit function cleanup { @@ -38,7 +38,7 @@ if mkdir $LOCKDIR; then # get new zeek logs ready for processing cd "$ZEEK_LOGS_DIR" - find . -path ./processed -prune -o -path ./current -prune -o -path ./upload -prune -o -path ./extract_files -prune -o -type f -exec file --separator '|' --mime-type "{}" \; | grep -P "(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)" | awk -F'|' '{print $1}' | sort -V | \ + find . -path ./processed -prune -o -path ./current -prune -o -path ./upload -prune -o -path ./extract_files -prune -o -path ./live -prune -o -type f -exec file --separator '|' --mime-type "{}" \; | grep -P "(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)" | awk -F'|' '{print $1}' | sort -V | \ xargs -n 1 -P $FILEBEAT_PREPARE_PROCESS_COUNT -I '{}' bash -c ' fuser -s "{}" 2>/dev/null diff --git a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.sh b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.sh index 19597a7ee..afcc22be9 100755 --- a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.sh +++ b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.sh @@ -3,11 +3,11 @@ # Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved. -PROCESS_DIR=${FILEBEAT_ZEEK_DIR:-/data/zeek/} +PROCESS_DIR=${FILEBEAT_ZEEK_DIR:-/zeek/} UPLOAD_DIR="${PROCESS_DIR}/upload" mkdir -p "$UPLOAD_DIR" -# as new zeek log archives are closed for writing in /data/zeek/upload, move them to /data/zeek for processing +# as new zeek log archives are closed for writing in /zeek/upload, move them to /zeek for processing inotifywait -m -e close_write --format '%w%f' "${UPLOAD_DIR}" | while read NEWFILE do FILEMIME=$(file -b --mime-type "$NEWFILE") diff --git a/filebeat/supervisord.conf b/filebeat/supervisord.conf index db85c13be..2b660ce10 100644 --- a/filebeat/supervisord.conf +++ b/filebeat/supervisord.conf @@ -29,7 +29,7 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:filebeat-nginx] -command=bash -c "/data/opensearch_status.sh && /usr/local/bin/docker-entrypoint -e --strict.perms=false \ +command=bash -c "/usr/local/bin/opensearch_status.sh && /usr/local/bin/docker-entrypoint -e --strict.perms=false \ --path.home /usr/share/filebeat-nginx \ --path.config /usr/share/filebeat-nginx \ --path.data /usr/share/filebeat-nginx/data \ @@ -47,7 +47,7 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:filebeat-tcp] -command=bash -c "/data/opensearch_status.sh && /usr/local/bin/docker-entrypoint -e --strict.perms=false \ +command=bash -c "/usr/local/bin/opensearch_status.sh && /usr/local/bin/docker-entrypoint -e --strict.perms=false \ --path.home /usr/share/filebeat-tcp \ --path.config /usr/share/filebeat-tcp \ --path.data /usr/share/filebeat-tcp/data \ @@ -64,13 +64,12 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:watch-upload] -command=/bin/bash -c "sleep 30 && /data/filebeat-watch-zeeklogs-uploads-folder.sh" +command=/bin/bash -c "sleep 30 && /usr/local/bin/filebeat-watch-zeeklogs-uploads-folder.sh" user=root startsecs=35 startretries=1 stopasgroup=true killasgroup=true -directory=/data stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true diff --git a/logstash/pipelines/suricata/01_input_suricata.conf b/logstash/pipelines/suricata/01_input_suricata.conf index 207fcff52..f280362d9 100644 --- a/logstash/pipelines/suricata/01_input_suricata.conf +++ b/logstash/pipelines/suricata/01_input_suricata.conf @@ -6,9 +6,10 @@ input { filter { # this pipeline only needs to see suricata logs forwarded from filebeat - if ("_filebeat_suricata" in [tags]) { + if ("_filebeat_suricata" in [tags]) or ("_filebeat_suricata_live" in [tags]) { mutate { id => "mutate_filebeat_suricata_forward_tag_remove" - remove_tag => [ "_filebeat_suricata" ] } + remove_tag => [ "_filebeat_suricata", + "_filebeat_suricata_live" ] } } else { drop { id => "drop_not_filebeat_suricata" } } diff --git a/logstash/pipelines/suricata/11_suricata_logs.conf b/logstash/pipelines/suricata/11_suricata_logs.conf index 5706cce09..c8a17260d 100644 --- a/logstash/pipelines/suricata/11_suricata_logs.conf +++ b/logstash/pipelines/suricata/11_suricata_logs.conf @@ -21,7 +21,7 @@ filter { tags = fileParts.captures[0] unless tags.nil? filenameTags = tags.split(',') - filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(eve|pcap|dmp|log|bro|zeek|suricata|tcpdump|netsniff|autozeek|autosuricata)s?\s*\z/i))} + filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(eve|pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i))} event.set('[@metadata][suricata_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0) end end" diff --git a/logstash/pipelines/zeek/01_input_zeek.conf b/logstash/pipelines/zeek/01_input_zeek.conf index a9c706461..b09fbcff6 100644 --- a/logstash/pipelines/zeek/01_input_zeek.conf +++ b/logstash/pipelines/zeek/01_input_zeek.conf @@ -6,9 +6,10 @@ input { filter { # this pipeline only needs to see Zeek logs forwarded from filebeat - if ("_filebeat_zeek" in [tags]) { + if ("_filebeat_zeek" in [tags]) or ("_filebeat_zeek_live" in [tags]) { mutate { id => "mutate_filebeat_zeek_forward_tag_remove" - remove_tag => [ "_filebeat_zeek" ] } + remove_tag => [ "_filebeat_zeek", + "_filebeat_zeek_live" ] } } else { drop { id => "drop_not_filebeat_zeek" } } diff --git a/logstash/pipelines/zeek/11_zeek_logs.conf b/logstash/pipelines/zeek/11_zeek_logs.conf index e264a577c..b0a6062e0 100644 --- a/logstash/pipelines/zeek/11_zeek_logs.conf +++ b/logstash/pipelines/zeek/11_zeek_logs.conf @@ -64,7 +64,7 @@ filter { event.set('[@metadata][zeek_fields_bitmap]', zeekFieldsTags[2].to_i(16)) end end - filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(ZEEKFLDx|autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|tcpdump|netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} + filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(ZEEKFLDx|autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} event.set('[@metadata][zeek_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0) " # diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index bcc1dc773..351602843 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -103,12 +103,13 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/pcap/processed/" mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/" mkdir -p "$MALCOLM_DEST_DIR/scripts/" - mkdir -p "$MALCOLM_DEST_DIR/suricata-logs/" + mkdir -p "$MALCOLM_DEST_DIR/suricata-logs/live" mkdir -p "$MALCOLM_DEST_DIR/suricata/rules/" mkdir -p "$MALCOLM_DEST_DIR/yara/rules/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/current/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/extract_files/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/" + mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/live/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX" diff --git a/pcap-capture/templates/netsniff.template b/pcap-capture/templates/netsniff.template index b69c083e3..3cd1e5a6c 100644 --- a/pcap-capture/templates/netsniff.template +++ b/pcap-capture/templates/netsniff.template @@ -1,5 +1,5 @@ [program:netsniff-$IFACE] -command=/usr/sbin/netsniff-ng -i "$IFACE" -T "%(ENV_PCAP_NETSNIFF_MAGIC)s" -o "%(ENV_PCAP_PATH)s" -P "netsniff-$IFACE_" -F "%(ENV_PCAP_ROTATE_MEGABYTES)sMiB" --silent "%(ENV_PCAP_FILTER)s" +command=/usr/sbin/netsniff-ng -i "$IFACE" -T "%(ENV_PCAP_NETSNIFF_MAGIC)s" -o "%(ENV_PCAP_PATH)s" -P "mnetsniff-$IFACE_" -F "%(ENV_PCAP_ROTATE_MEGABYTES)sMiB" --silent "%(ENV_PCAP_FILTER)s" user=%(ENV_PUSER)s startsecs=5 startretries=3 diff --git a/pcap-capture/templates/tcpdump.template b/pcap-capture/templates/tcpdump.template index 694aae903..cc7c4b3cd 100644 --- a/pcap-capture/templates/tcpdump.template +++ b/pcap-capture/templates/tcpdump.template @@ -1,5 +1,5 @@ [program:tcpdump-$IFACE] -command=/usr/bin/tcpdump -i "$IFACE" -s %(ENV_PCAP_SNAPLEN)s -w "tcpdump-$IFACE_%(ENV_PCAP_TCPDUMP_FILENAME_PATTERN)s" -G %(ENV_PCAP_ROTATE_SECONDS)s -C %(ENV_PCAP_ROTATE_MEGABYTES)s -K -n "%(ENV_PCAP_FILTER)s" +command=/usr/bin/tcpdump -i "$IFACE" -s %(ENV_PCAP_SNAPLEN)s -w "mtcpdump-$IFACE_%(ENV_PCAP_TCPDUMP_FILENAME_PATTERN)s" -G %(ENV_PCAP_ROTATE_SECONDS)s -C %(ENV_PCAP_ROTATE_MEGABYTES)s -K -n "%(ENV_PCAP_FILTER)s" user=%(ENV_PUSER)s startsecs=5 startretries=3 diff --git a/scripts/control.py b/scripts/control.py index 2a249c2c5..67f157550 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -473,7 +473,12 @@ def stop(wipe=False): if wipe: # delete OpenSearch database - shutil.rmtree(os.path.join(MalcolmPath, 'opensearch/nodes'), ignore_errors=True) + shutil.rmtree(os.path.join(MalcolmPath, os.path.join('opensearch', 'nodes')), ignore_errors=True) + + # delete Zeek live-related spool files + shutil.rmtree( + os.path.join(MalcolmPath, os.path.join('zeek-logs', os.path.join('live', 'spool'))), ignore_errors=True + ) # delete data files (backups, zeek logs, arkime logs, PCAP files, captured PCAP files) for dataDir in ['opensearch-backup', 'zeek-logs', 'suricata-logs', 'arkime-logs', 'pcap', 'arkime-raw']: @@ -491,6 +496,8 @@ def stop(wipe=False): os.path.join('opensearch-backup', 'logs'), os.path.join('zeek-logs', 'processed'), os.path.join('zeek-logs', 'current'), + os.path.join('zeek-logs', 'live'), + os.path.join('suricata-logs'), ]: RemoveEmptyFolders(dataDir, removeRoot=False) @@ -544,10 +551,11 @@ def start(): os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), os.path.join(MalcolmPath, os.path.join('pcap', 'processed')), os.path.join(MalcolmPath, os.path.join('pcap', 'upload')), - os.path.join(MalcolmPath, os.path.join('suricata-logs')), + os.path.join(MalcolmPath, os.path.join('suricata-logs', 'live')), os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'MISP'))), os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'STIX'))), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'current')), + os.path.join(MalcolmPath, os.path.join('zeek-logs', 'live')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'extract_files')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'processed')), os.path.join(MalcolmPath, os.path.join('zeek-logs', 'upload')), diff --git a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh index e523dcb22..fe9e13d84 100755 --- a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh +++ b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh @@ -576,7 +576,7 @@ function InstallMalcolm { done done touch auth.env - grep image: docker-compose-standalone.yml | awk '{print $2}' | xargs -l -r $SUDO_CMD docker pull + grep image: docker-compose-standalone.yml | awk '{print $2}' | sort -u | xargs -l -r $SUDO_CMD docker pull echo "Please run $MALCOLM_PATH/scripts/auth_setup to complete configuration" >&2 popd >/dev/null 2>&1 fi diff --git a/scripts/github_image_helper.sh b/scripts/github_image_helper.sh index 9a8a88739..dd4aa2ec4 100755 --- a/scripts/github_image_helper.sh +++ b/scripts/github_image_helper.sh @@ -80,7 +80,7 @@ function PullAndTagGithubWorkflowImages() { VERSION="$(_malcolmversion)" OWNER="$(_gitowner)" echo "Pulling images from ghcr.io/$OWNER ($BRANCH) and tagging as $VERSION ..." - for IMG in $(grep image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1); do + for IMG in $(grep image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1 | sort -u); do _PullAndTagGithubWorkflowBuild "$IMG" done echo "done" diff --git a/scripts/install.py b/scripts/install.py index 84e21d371..f3e6d2886 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -507,8 +507,8 @@ def tweak_malcolm_runtime( 'Determine oldest indices by name (instead of creation time)?', default=True ) - autoSuricata = InstallerYesOrNo('Automatically analyze all PCAP files with Suricata?', default=True) - autoZeek = InstallerYesOrNo('Automatically analyze all PCAP files with Zeek?', default=True) + autoSuricata = InstallerYesOrNo('Automatically analyze all uploaded PCAP files with Suricata?', default=True) + autoZeek = InstallerYesOrNo('Automatically analyze all uploaded PCAP files with Zeek?', default=True) reverseDns = InstallerYesOrNo( 'Perform reverse DNS lookup locally for source and destination IP addresses in logs?', default=False ) @@ -583,16 +583,26 @@ def tweak_malcolm_runtime( # input packet capture parameters pcapNetSniff = False pcapTcpDump = False + liveZeek = False + liveSuricata = False pcapIface = 'lo' pcapFilter = '' - if InstallerYesOrNo('Should Malcolm capture network traffic to PCAP files?', default=False): + + if InstallerYesOrNo( + 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', default=False + ): + pcapNetSniff = InstallerYesOrNo('Capture packets using netsniff-ng?', default=True) + pcapTcpDump = InstallerYesOrNo('Capture packets using tcpdump?', default=(not pcapNetSniff)) + + liveSuricata = InstallerYesOrNo('Should Malcolm analyze live network traffic with Suricata?', default=False) + liveZeek = InstallerYesOrNo('Should Malcolm analyze live network traffic with Zeek?', default=False) + + if pcapNetSniff or pcapTcpDump or liveZeek or liveSuricata: pcapIface = '' while len(pcapIface) <= 0: pcapIface = InstallerAskForString('Specify capture interface(s) (comma-separated)') - pcapNetSniff = InstallerYesOrNo('Capture packets using netsniff-ng?', default=True) - pcapTcpDump = InstallerYesOrNo('Capture packets using tcpdump?', default=(not pcapNetSniff)) pcapFilter = InstallerAskForString( - 'PCAP capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', default='' + 'Capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', default='' ) # modify specified values in-place in docker-compose files @@ -725,6 +735,18 @@ def tweak_malcolm_runtime( r'(PCAP_ENABLE_TCPDUMP\s*:\s*)(\S+)', fr"\g<1>{TrueOrFalseQuote(pcapTcpDump)}", line ) + elif 'ZEEK_LIVE_CAPTURE' in line: + # live traffic analysis with Zeek + line = re.sub( + r'(ZEEK_LIVE_CAPTURE\s*:\s*)(\S+)', fr"\g<1>{TrueOrFalseQuote(liveZeek)}", line + ) + + elif 'SURICATA_LIVE_CAPTURE' in line: + # live traffic analysis with Suricata + line = re.sub( + r'(SURICATA_LIVE_CAPTURE\s*:\s*)(\S+)', fr"\g<1>{TrueOrFalseQuote(liveSuricata)}", line + ) + elif 'PCAP_IFACE' in line: # capture interface(s) line = re.sub(r'(PCAP_IFACE\s*:\s*)(\S+)', fr"\g<1>'{pcapIface}'", line) @@ -734,7 +756,7 @@ def tweak_malcolm_runtime( line = re.sub(r'(PCAP_FILTER\s*:)(.*)', fr"\g<1> '{pcapFilter}'", line) elif 'ZEEK_AUTO_ANALYZE_PCAP_FILES' in line: - # automatic pcap analysis with Zeek + # automatic uploaded pcap analysis with Zeek line = re.sub( r'(ZEEK_AUTO_ANALYZE_PCAP_FILES\s*:\s*)(\S+)', fr"\g<1>{TrueOrFalseQuote(autoZeek)}", @@ -742,7 +764,7 @@ def tweak_malcolm_runtime( ) elif 'SURICATA_AUTO_ANALYZE_PCAP_FILES' in line: - # automatic pcap analysis with suricata + # automatic uploaded pcap analysis with suricata line = re.sub( r'(SURICATA_AUTO_ANALYZE_PCAP_FILES\s*:\s*)(\S+)', fr"\g<1>{TrueOrFalseQuote(autoSuricata)}", diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index cb9f666f0..c6e8e6c67 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -74,11 +74,12 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/pcap/processed/" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/" mkdir $VERBOSE -p "$DESTDIR/scripts/" - mkdir $VERBOSE -p "$DESTDIR/suricata-logs/" + mkdir $VERBOSE -p "$DESTDIR/suricata-logs/live" mkdir $VERBOSE -p "$DESTDIR/suricata/rules/" mkdir $VERBOSE -p "$DESTDIR/yara/rules/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/current/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/extract_files/" + mkdir $VERBOSE -p "$DESTDIR/zeek-logs/live/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/processed/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/upload/" mkdir $VERBOSE -p "$DESTDIR/zeek/intel/MISP" @@ -125,7 +126,7 @@ if mkdir "$DESTDIR"; then if [[ $CONFIRMATION =~ ^[Yy]$ ]]; then echo "This might take a few minutes..." DESTNAMEIMAGES="$RUN_PATH/$(basename $DESTDIR)_images.tar.gz" - IMAGES=( $(grep image: $DESTDIR/docker-compose.yml | awk '{print $2}') ) + IMAGES=( $(grep image: $DESTDIR/docker-compose.yml | awk '{print $2}' | sort -u) ) docker save "${IMAGES[@]}" | gzip > "$DESTNAMEIMAGES" echo "Packaged Malcolm docker images to \"$DESTNAMEIMAGES\"" echo "" diff --git a/sensor-iso/README.md b/sensor-iso/README.md index 1837e4cf7..9cbd6eef6 100644 --- a/sensor-iso/README.md +++ b/sensor-iso/README.md @@ -397,7 +397,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-6.1.1.iso" +Finished, created "/sensor-build/hedgehog-6.2.0.iso" … ``` diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index f1e5e0471..8d2bab747 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -107,6 +107,7 @@ if [ -d "$WORKDIR" ]; then # copy shared scripts rsync -a "$SCRIPT_PATH/shared/bin/" ./config/includes.chroot/usr/local/bin/ + mv ./config/includes.chroot/usr/local/bin/zeekdeploy.sh /opt/zeek/bin/ chown -R root:root ./config/includes.chroot/usr/local/bin/ # write out some version stuff specific to this installation version diff --git a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml index c4c654607..0a559fd09 100644 --- a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml +++ b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml @@ -9,7 +9,7 @@ filebeat.inputs: - ${BEAT_STATIC_LOG_PATTERN:/home/sensor/bro_logs/static/*.log} symlinks: true fields_under_root: true - tags: ["_filebeat_zeek"] + tags: ["_filebeat_zeek_live"] compression_level: 0 exclude_lines: ['^\s*#'] scan_frequency: ${BEAT_SCAN_FREQUENCY:10s} @@ -27,7 +27,7 @@ filebeat.inputs: - ${BEAT_SURICATA_LOG_PATTERN:/home/sensor/bro_logs/suricata/eve*.json} symlinks: true fields_under_root: true - tags: ["_filebeat_suricata"] + tags: ["_filebeat_suricata_live"] compression_level: 0 scan_frequency: ${BEAT_SCAN_FREQUENCY:10s} clean_inactive: ${BEAT_CLEAN_INACTIVE:180m} diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index d7f7a3320..6ba968a29 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -193,7 +193,7 @@ def zeekFileWorker(zeekWorkerArgs): scanWorkerId = scanWorkersCount.increment() # unique ID for this thread - newFileQueue, pcapBaseDir, zeekBin, autoZeek, autoTag, uploadDir, defaultExtractFileMode = ( + newFileQueue, pcapBaseDir, zeekBin, autoZeek, forceZeek, autoTag, uploadDir, defaultExtractFileMode = ( zeekWorkerArgs[0], zeekWorkerArgs[1], zeekWorkerArgs[2], @@ -201,6 +201,7 @@ def zeekFileWorker(zeekWorkerArgs): zeekWorkerArgs[4], zeekWorkerArgs[5], zeekWorkerArgs[6], + zeekWorkerArgs[7], ) if debug: @@ -220,9 +221,20 @@ def zeekFileWorker(zeekWorkerArgs): fileInfo[FILE_INFO_DICT_NAME] = os.path.join(pcapBaseDir, fileInfo[FILE_INFO_DICT_NAME]) if os.path.isfile(fileInfo[FILE_INFO_DICT_NAME]): - # zeek this PCAP if it's tagged "AUTOZEEK" or if the global autoZeek flag is turned on - if autoZeek or ( - (FILE_INFO_DICT_TAGS in fileInfo) and ZEEK_AUTOZEEK_TAG in fileInfo[FILE_INFO_DICT_TAGS] + # Zeek this PCAP if it's tagged "AUTOZEEK" or if the global autoZeek flag is turned on. + # However, skip "live" PCAPs Malcolm is capturing and rotating through for Arkime capture, + # as Zeek now does its own network capture in Malcolm standalone mode. + if ( + autoZeek + or ((FILE_INFO_DICT_TAGS in fileInfo) and ZEEK_AUTOZEEK_TAG in fileInfo[FILE_INFO_DICT_TAGS]) + ) and ( + forceZeek + or ( + not any( + os.path.basename(fileInfo[FILE_INFO_DICT_NAME]).startswith(prefix) + for prefix in ('mnetsniff', 'mtcpdump') + ) + ) ): extractFileMode = defaultExtractFileMode @@ -337,7 +349,7 @@ def suricataFileWorker(suricataWorkerArgs): scanWorkerId = scanWorkersCount.increment() # unique ID for this thread - newFileQueue, pcapBaseDir, autoSuricata, suricataBin, autoTag, uploadDir, suricataConfig = ( + newFileQueue, pcapBaseDir, autoSuricata, forceSuricata, suricataBin, autoTag, uploadDir, suricataConfig = ( suricataWorkerArgs[0], suricataWorkerArgs[1], suricataWorkerArgs[2], @@ -345,6 +357,7 @@ def suricataFileWorker(suricataWorkerArgs): suricataWorkerArgs[4], suricataWorkerArgs[5], suricataWorkerArgs[6], + suricataWorkerArgs[7], ) if debug: @@ -360,9 +373,22 @@ def suricataFileWorker(suricataWorkerArgs): else: if isinstance(fileInfo, dict) and (FILE_INFO_DICT_NAME in fileInfo): - # suricata this PCAP if it's tagged "AUTOSURICATA" or if the global autoSuricata flag is turned on - if autoSuricata or ( - (FILE_INFO_DICT_TAGS in fileInfo) and SURICATA_AUTOSURICATA_TAG in fileInfo[FILE_INFO_DICT_TAGS] + # Suricata this PCAP if it's tagged "AUTOSURICATA" or if the global autoSuricata flag is turned on. + # However, skip "live" PCAPs Malcolm is capturing and rotating through for Arkime capture, + # as Suricata now does its own network capture in Malcolm standalone mode. + if ( + autoSuricata + or ( + (FILE_INFO_DICT_TAGS in fileInfo) and SURICATA_AUTOSURICATA_TAG in fileInfo[FILE_INFO_DICT_TAGS] + ) + ) and ( + forceSuricata + or ( + not any( + os.path.basename(fileInfo[FILE_INFO_DICT_NAME]).startswith(prefix) + for prefix in ('mnetsniff', 'mtcpdump') + ) + ) ): if pcapBaseDir and os.path.isdir(pcapBaseDir): @@ -589,6 +615,17 @@ def main(): default=False, required=False, ) + parser.add_argument( + '--forcezeek', + dest='forceZeek', + help="Force Zeek analysis even on rotated PCAPs", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) parser.add_argument( '--extract', dest='zeekExtractFileMode', @@ -626,6 +663,17 @@ def main(): default=False, required=False, ) + parser.add_argument( + '--forcesuricata', + dest='forceSuricata', + help="Force Suricata analysis even on rotated PCAPs", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) requiredNamed.add_argument( '--suricata-config', dest='suricataConfigFile', @@ -711,6 +759,7 @@ def main(): args.pcapBaseDir, args.executable, args.autoZeek, + args.forceZeek, args.autoTag, args.zeekUploadDir, args.zeekExtractFileMode, @@ -726,6 +775,7 @@ def main(): newFileQueue, args.pcapBaseDir, args.autoSuricata, + args.forceSuricata, args.executable, args.autoTag, args.suricataUploadDir, diff --git a/shared/bin/pcap_utils.py b/shared/bin/pcap_utils.py index 896175396..2d12802d3 100644 --- a/shared/bin/pcap_utils.py +++ b/shared/bin/pcap_utils.py @@ -147,5 +147,5 @@ def tags_from_filename(filespec): # split tags on these characters tagSplitterRe = "[,-/_.]+" # tags to ignore explicitly - regex = re.compile(r'^(\d+|p?cap|dmp|log|bro|zeek|tcpdump|netsniff)$', re.IGNORECASE) + regex = re.compile(r'^(\d+|p?cap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff)$', re.IGNORECASE) return list(filter(lambda i: not regex.search(i), map(str.strip, filter(None, re.split(tagSplitterRe, filespec))))) diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index e23f8f0bc..c24e736d8 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -356,7 +356,8 @@ def ObjToYamlStrLines(obj, options=None): for varName, varVal in [ (key.upper(), value) for key, value in os.environ.items() - if key.upper().startswith('SURICATA') or key.upper() in ('CAPTURE_INTERFACE', 'CAPTURE_FILTER', 'SUPERVISOR_PATH') + if key.upper().startswith('SURICATA') + or key.upper() in ('CAPTURE_INTERFACE', 'CAPTURE_FILTER', 'PCAP_IFACE', 'PCAP_FILTER', 'SUPERVISOR_PATH') ]: tmpYaml = YAML(typ='safe') newVal = tmpYaml.load(varVal) @@ -801,11 +802,16 @@ def main(): ) # af-packet interface definitions - if DEFAULT_VARS['CAPTURE_INTERFACE'] is not None: + captureIface = ( + DEFAULT_VARS['CAPTURE_INTERFACE'] + if DEFAULT_VARS['CAPTURE_INTERFACE'] is not None + else DEFAULT_VARS['PCAP_IFACE'] + ) + if captureIface is not None: cfg.pop('af-packet', None) cfg['af-packet'] = [{'interface': 'default'}] clusterId = 99 - for iface in DEFAULT_VARS['CAPTURE_INTERFACE'].split(','): + for iface in captureIface.split(','): cfg['af-packet'].insert( 0, { @@ -813,7 +819,9 @@ def main(): 'cluster-id': clusterId, 'block-size': DEFAULT_VARS['AF_PACKET_BLOCK_SIZE'], 'block-timeout': DEFAULT_VARS['AF_PACKET_BLOCK_TIMEOUT'], - 'bpf-filter': DEFAULT_VARS['CAPTURE_FILTER'], + 'bpf-filter': DEFAULT_VARS['CAPTURE_FILTER'] + if DEFAULT_VARS['CAPTURE_FILTER'] is not None + else DEFAULT_VARS['PCAP_FILTER'], 'buffer-size': DEFAULT_VARS['AF_PACKET_BUFFER_SIZE'], 'checksum-checks': DEFAULT_VARS['AF_PACKET_CHECKSUM_CHECKS'], 'cluster-type': DEFAULT_VARS['AF_PACKET_CLUSTER_TYPE'], diff --git a/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh b/shared/bin/zeekdeploy.sh similarity index 91% rename from sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh rename to shared/bin/zeekdeploy.sh index 70ea9dc50..b4ba6c8a8 100755 --- a/sensor-iso/config/includes.chroot/opt/zeek/bin/zeekdeploy.sh +++ b/shared/bin/zeekdeploy.sh @@ -26,14 +26,22 @@ else fi fi +if [[ -z "$CAPTURE_INTERFACE" ]] && [[ -n "$PCAP_IFACE" ]]; then + CAPTURE_INTERFACE="$PCAP_IFACE" +fi +if [[ -z "$CAPTURE_FILTER" ]] && [[ -n "$PCAP_FILTER" ]]; then + CAPTURE_FILTER="$PCAP_FILTER" +fi + # capture interface(s) *must* be specified if [[ -z $CAPTURE_INTERFACE ]] ; then echo "Zeek capture interface(s) (via \$CAPTURE_INTERFACE) not specified" exit 1 fi -# do we have AF_PACKET support in the kernel? true if > 0 -AF_PACKET_SUPPORT=$(grep -c -x 'CONFIG_PACKET=[ym]' "/boot/config-$(uname -r)") +# do we have AF_PACKET support in the kernel? if we can't determine, assume "yes" +BOOT_CONFIG_FILE="/boot/config-$(uname -r)" +[[ -r "$BOOT_CONFIG_FILE" ]] && AF_PACKET_SUPPORT=$(grep -c -x 'CONFIG_PACKET=[ym]' "$BOOT_CONFIG_FILE") || AF_PACKET_SUPPORT=1 # determine location of zeekctl script and relative installation path ZEEK_CTL="$(which zeekctl)" @@ -55,6 +63,7 @@ fi # some other defaults [[ -z $ZEEK_LB_PROCS ]] && ZEEK_LB_PROCS="1" +[[ -z $WORKER_LB_PROCS ]] && WORKER_LB_PROCS="$ZEEK_LB_PROCS" [[ -z $ZEEK_LB_METHOD ]] && ZEEK_LB_METHOD="custom" [[ -z $ZEEK_AF_PACKET_BUFFER_SIZE ]] && ZEEK_AF_PACKET_BUFFER_SIZE="$(echo "64*1024*1024" | bc)" @@ -64,7 +73,7 @@ ZEEK_LOG_PATH="$($REALPATH "$ZEEK_LOG_PATH")" ARCHIVE_PATH="$ZEEK_LOG_PATH/logs" WORK_PATH="$ZEEK_LOG_PATH/spool" TMP_PATH="$ZEEK_INSTALL_PATH/spool/tmp" -EXTRACT_FILES_PATH="$ZEEK_LOG_PATH/extract_files" +[[ -z $EXTRACT_FILES_PATH ]] && EXTRACT_FILES_PATH="$ZEEK_LOG_PATH/extract_files" mkdir -p "$ARCHIVE_PATH"/static "$WORK_PATH" "$EXTRACT_FILES_PATH" "$TMP_PATH" export TMP="$TMP_PATH" @@ -76,7 +85,8 @@ ZEEK_EXTRACTOR_SCRIPT="$ZEEK_INSTALL_PATH"/share/zeek/site/"$EXTRACTOR_ZEEK_SCRI ([[ ! -r "$ZEEK_EXTRACTOR_OVERRIDE_FILE" ]] || [[ -z "$ZEEK_EXTRACTOR_SCRIPT" ]] || [[ ! "$ZEEK_EXTRACTOR_MODE" = "mapped" ]]) && ZEEK_EXTRACTOR_OVERRIDE_FILE="" # make sure "intel" directory exists, even if empty -export INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel +[[ -n "$ZEEK_INTEL_PATH" ]] && INTEL_DIR="$ZEEK_INTEL_PATH" || INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel +export INTEL_DIR mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP touch "$INTEL_DIR"/__load__.zeek # autoconfigure load directives for intel files diff --git a/suricata-logs/live/.gitignore b/suricata-logs/live/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/suricata-logs/live/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/suricata/scripts/docker_entrypoint.sh b/suricata/scripts/docker_entrypoint.sh index af84b01d4..a3d3e0200 100755 --- a/suricata/scripts/docker_entrypoint.sh +++ b/suricata/scripts/docker_entrypoint.sh @@ -1,7 +1,17 @@ #!/bin/bash -# modify suricata.yaml according to environment variables -/usr/local/bin/suricata_config_populate.py ${SURICATA_TEST_CONFIG_VERBOSITY:-} >&2 +# ensure capabilities for capture +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata || true + +# modify suricata.yaml according to environment variables (as non-root) +if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then + su -s /bin/bash -p ${PUSER} << EOF + /usr/local/bin/suricata_config_populate.py ${SURICATA_TEST_CONFIG_VERBOSITY:-} >&2 +EOF +else + /usr/local/bin/suricata_config_populate.py ${SURICATA_TEST_CONFIG_VERBOSITY:-} >&2 +fi # start supervisor (which will spawn pcap-suricata, cron, etc.) or whatever the default command is exec "$@" diff --git a/suricata/supervisord.conf b/suricata/supervisord.conf index fd11ce7f4..5b13c88b6 100644 --- a/suricata/supervisord.conf +++ b/suricata/supervisord.conf @@ -1,3 +1,4 @@ +; Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) @@ -5,6 +6,7 @@ chmod=0700 [supervisord] nodaemon=true +user=root logfile=/dev/null logfile_maxbytes=0 pidfile=/tmp/supervisord.pid @@ -27,8 +29,10 @@ command=python3 /opt/pcap_suricata_processor.py --suricata /usr/bin/suricata --autotag "%(ENV_AUTO_TAG)s" --autosuricata "%(ENV_SURICATA_AUTO_ANALYZE_PCAP_FILES)s" + --forcesuricata "%(ENV_SURICATA_ROTATED_PCAP)s" --suricata-config "%(ENV_SURICATA_CONFIG_FILE)s" --suricata-directory "%(ENV_SURICATA_LOG_DIR)s" +autostart=%(ENV_SURICATA_PCAP_PROCESSOR)s startsecs=15 startretries=1 stopasgroup=true @@ -37,13 +41,34 @@ directory=/opt/suricata stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:cron] autorestart=true +autostart=%(ENV_SURICATA_CRON)s command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s" user=%(ENV_PUSER)s stopasgroup=true killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 -redirect_stderr=true \ No newline at end of file +redirect_stderr=true +user=%(ENV_PUSER)s + +[program:live-suricata] +command=/usr/bin/suricata + -v + -c "%(ENV_SURICATA_CONFIG_FILE)s" + -l "%(ENV_SURICATA_LOG_DIR)s/live" + --af-packet +autostart=%(ENV_SURICATA_LIVE_CAPTURE)s +autorestart=true +startsecs=180 +stopwaitsecs=15 +startretries=3 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=%(ENV_PUSER)s \ No newline at end of file diff --git a/zeek-logs/live/.gitignore b/zeek-logs/live/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/zeek-logs/live/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh new file mode 100755 index 000000000..077f6c222 --- /dev/null +++ b/zeek/scripts/docker_entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} + +# ensure capabilities for capture +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats || true + +if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then + if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then + su -s /bin/bash -p ${PUSER} << EOF + "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true +EOF + else + "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true + fi +fi + +# start supervisor (which will spawn pcap-zeek, cron, etc.) or whatever the default command is +exec "$@" diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf index bedd16ff6..e11aa1ee3 100644 --- a/zeek/supervisord.conf +++ b/zeek/supervisord.conf @@ -6,6 +6,7 @@ chmod=0700 [supervisord] nodaemon=true +user=root logfile=/dev/null logfile_maxbytes=0 pidfile=/tmp/supervisord.pid @@ -28,8 +29,11 @@ command=python3 /usr/local/bin/pcap_zeek_processor.py --zeek /opt/zeek/bin/zeek --autotag "%(ENV_AUTO_TAG)s" --autozeek "%(ENV_ZEEK_AUTO_ANALYZE_PCAP_FILES)s" + --forcezeek "%(ENV_ZEEK_ROTATED_PCAP)s" --extract "%(ENV_ZEEK_EXTRACTOR_MODE)s" --zeek-directory /zeek/upload +autostart=%(ENV_ZEEK_PCAP_PROCESSOR)s +autorestart=true startsecs=15 startretries=1 stopasgroup=true @@ -37,12 +41,29 @@ killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:cron] -autorestart=true command=/usr/local/bin/supercronic -json "%(ENV_SUPERCRONIC_CRONTAB)s" +autostart=%(ENV_ZEEK_CRON)s +autorestart=true +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=%(ENV_PUSER)s + +[program:live-zeek] +command=/opt/zeek/bin/zeekdeploy.sh +autostart=%(ENV_ZEEK_LIVE_CAPTURE)s +autorestart=true +startsecs=180 +stopwaitsecs=15 +startretries=3 stopasgroup=true killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s \ No newline at end of file