diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 1ac169ecd..094676f21 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -40,6 +40,8 @@ ARG EXTRACTED_FILE_ENABLE_FRESHCLAM=false ARG EXTRACTED_FILE_PIPELINE_DEBUG=false ARG EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA=false ARG CLAMD_SOCKET_FILE=/tmp/clamd.ctl +ARG EXTRACTED_FILE_ENABLE_YARA=false +ARG EXTRACTED_FILE_YARA_CUSTOM_ONLY=false ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH ENV ZEEK_LOG_DIRECTORY $ZEEK_LOG_DIRECTORY @@ -60,16 +62,34 @@ ENV EXTRACTED_FILE_ENABLE_FRESHCLAM $EXTRACTED_FILE_ENABLE_FRESHCLAM ENV EXTRACTED_FILE_PIPELINE_DEBUG $EXTRACTED_FILE_PIPELINE_DEBUG ENV EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA $EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA ENV CLAMD_SOCKET_FILE $CLAMD_SOCKET_FILE +ENV EXTRACTED_FILE_ENABLE_YARA $EXTRACTED_FILE_ENABLE_YARA +ENV EXTRACTED_FILE_YARA_CUSTOM_ONLY $EXTRACTED_FILE_YARA_CUSTOM_ONLY +ENV YARA_VERSION "4.0.2" +ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz" +ENV YARA_RULES_URL "https://codeload.github.com/Neo23x0/signature-base/tar.gz/master" +ENV YARA_RULES_DIR "/yara-rules" +ENV SRC_BASE_DIR "/usr/local/src" RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \ apt-get update && \ apt-get install --no-install-recommends -y -q \ + automake \ bc \ clamav \ clamav-daemon \ clamav-freshclam \ + curl \ + gcc \ libclamunrar9 \ - wget && \ + libjansson-dev \ + libjansson4 \ + libmagic-dev \ + libmagic1 \ + libssl-dev \ + libssl1.1 \ + libtool \ + make \ + pkg-config && \ apt-get -y -q install \ inotify-tools \ libzmq5 \ @@ -81,14 +101,46 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list python3-pyinotify \ python3-requests \ python3-zmq && \ - pip3 install clamd supervisor && \ - apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove python3-dev build-essential && \ + pip3 install clamd supervisor yara-python && \ + mkdir -p "${SRC_BASE_DIR}" && \ + cd "${SRC_BASE_DIR}" && \ + curl -sSL "${YARA_URL}" | tar xzf - -C "${SRC_BASE_DIR}" && \ + cd "./yara-${YARA_VERSION}" && \ + ./bootstrap.sh && \ + ./configure --prefix=/usr \ + --with-crypto \ + --enable-magic \ + --enable-cuckoo \ + --enable-dotnet && \ + make && \ + make install && \ + cd /tmp && \ + rm -rf "${SRC_BASE_DIR}"/yara* && \ + mkdir -p ./Neo23x0 && \ + curl -sSL "$YARA_RULES_URL" | tar xzvf - -C ./Neo23x0 --strip-components 1 && \ + mkdir -p "${YARA_RULES_DIR}" && \ + cp ./Neo23x0/yara/* ./Neo23x0/vendor/yara/* "${YARA_RULES_DIR}"/ && \ + cp ./Neo23x0/LICENSE "${YARA_RULES_DIR}"/_LICENSE && \ + rm -rf /tmp/Neo23x0 && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \ + automake \ + build-essential \ + gcc \ + gcc-8 \ + libc6-dev \ + libgcc-8-dev \ + libjansson-dev \ + libmagic-dev \ + libssl-dev \ + libtool \ + make \ + python3-dev && \ apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - wget -O /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \ - wget -O /var/lib/clamav/daily.cvd http://database.clamav.net/daily.cvd && \ - wget -O /var/lib/clamav/bytecode.cvd http://database.clamav.net/bytecode.cvd && \ + curl -s -S -L -o /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \ + curl -s -S -L -o /var/lib/clamav/daily.cvd http://database.clamav.net/daily.cvd && \ + curl -s -S -L -o /var/lib/clamav/bytecode.cvd http://database.clamav.net/bytecode.cvd && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \ usermod -a -G tty ${PUSER} && \ @@ -105,12 +157,17 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list if ! [ -z $HTTPProxyServer ]; then echo "HTTPProxyServer $HTTPProxyServer" >> /etc/clamav/freshclam.conf; fi && \ if ! [ -z $HTTPProxyPort ]; then echo "HTTPProxyPort $HTTPProxyPort" >> /etc/clamav/freshclam.conf; fi && \ sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/freshclam.conf && \ - sed -i "s/^DatabaseOwner .*$/DatabaseOwner ${PUSER}/g" /etc/clamav/freshclam.conf + sed -i "s/^DatabaseOwner .*$/DatabaseOwner ${PUSER}/g" /etc/clamav/freshclam.conf && \ + ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/vtot_scan.py && \ + ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \ + ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \ + ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/malass_scan.py ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ ADD shared/bin/zeek_carve_*.py /usr/local/bin/ ADD shared/bin/malass_client.py /usr/local/bin/ ADD file-monitor/supervisord.conf /etc/supervisord.conf +ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh WORKDIR /data/zeek/extract_files @@ -118,7 +175,7 @@ VOLUME ["/var/lib/clamav"] EXPOSE 3310 -ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"] +ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"] CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"] diff --git a/Dockerfiles/freq.Dockerfile b/Dockerfiles/freq.Dockerfile index 24ae6bc5b..60a8833e0 100644 --- a/Dockerfiles/freq.Dockerfile +++ b/Dockerfiles/freq.Dockerfile @@ -48,7 +48,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ chown -R ${PUSER}:${PGROUP} /opt/freq_server && \ usermod -a -G tty ${PUSER} && \ - apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove git python3-dev && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove git python3-dev build-essential && \ apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile index 8822a857b..2fda87d75 100644 --- a/Dockerfiles/moloch.Dockerfile +++ b/Dockerfiles/moloch.Dockerfile @@ -4,7 +4,7 @@ FROM debian:buster-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV MOLOCH_VERSION "2.3.2" +ENV MOLOCH_VERSION "2.4.0" ENV MOLOCHDIR "/data/moloch" ENV MOLOCH_URL "https://codeload.github.com/aol/moloch/tar.gz/v${MOLOCH_VERSION}" ENV MOLOCH_LOCALELASTICSEARCH no @@ -28,7 +28,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list g++ \ gcc \ gettext \ - git \ + git-core \ groff \ groff-base \ imagemagick \ @@ -42,9 +42,10 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list libwww-perl \ libyaml-dev \ make \ + meson \ + ninja-build \ pandoc \ patch \ - python-dev \ python3-dev \ python3-pip \ python3-setuptools \ diff --git a/README.md b/README.md index 943e0ddf5..e24439f33 100644 --- a/README.md +++ b/README.md @@ -134,44 +134,44 @@ You must run [`auth_setup`](#AuthSetup) prior to pulling Malcolm's Docker images Malcolm's Docker images are periodically built and hosted on [Docker Hub](https://hub.docker.com/u/malcolmnetsec). If you already have [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/), these prebuilt images can be pulled by navigating into the Malcolm directory (containing the `docker-compose.yml` file) and running `docker-compose pull` like this: ``` $ docker-compose pull -Pulling curator ... done -Pulling elastalert ... done -Pulling elasticsearch ... done -Pulling file-monitor ... done -Pulling filebeat ... done -Pulling freq ... done -Pulling htadmin ... done -Pulling kibana ... done -Pulling logstash ... done -Pulling moloch ... done -Pulling name-map-ui ... done -Pulling nginx-proxy ... done -Pulling pcap-capture ... done -Pulling pcap-monitor ... done -Pulling upload ... done -Pulling zeek ... done +Pulling curator ... done +Pulling elastalert ... done +Pulling elasticsearch ... done +Pulling file-monitor ... done +Pulling filebeat ... done +Pulling freq ... done +Pulling htadmin ... done +Pulling kibana ... done +Pulling logstash ... done +Pulling moloch ... done +Pulling name-map-ui ... done +Pulling nginx-proxy ... done +Pulling pcap-capture ... done +Pulling pcap-monitor ... done +Pulling upload ... done +Pulling zeek ... done ``` You can then observe that the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/curator 2.2.1 xxxxxxxxxxxx 20 hours ago 246MB -malcolmnetsec/elastalert 2.2.1 xxxxxxxxxxxx 20 hours ago 408MB -malcolmnetsec/elasticsearch-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 693MB -malcolmnetsec/filebeat-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 474MB -malcolmnetsec/file-monitor 2.2.1 xxxxxxxxxxxx 20 hours ago 386MB -malcolmnetsec/file-upload 2.2.1 xxxxxxxxxxxx 20 hours ago 199MB -malcolmnetsec/freq 2.2.1 xxxxxxxxxxxx 20 hours ago 390MB -malcolmnetsec/htadmin 2.2.1 xxxxxxxxxxxx 20 hours ago 180MB -malcolmnetsec/kibana-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 1.07GB -malcolmnetsec/logstash-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 1.05GB -malcolmnetsec/moloch 2.2.1 xxxxxxxxxxxx 20 hours ago 667MB -malcolmnetsec/name-map-ui 2.2.1 xxxxxxxxxxxx 20 hours ago 134MB -malcolmnetsec/nginx-proxy 2.2.1 xxxxxxxxxxxx 20 hours ago 118MB -malcolmnetsec/pcap-capture 2.2.1 xxxxxxxxxxxx 20 hours ago 111MB -malcolmnetsec/pcap-monitor 2.2.1 xxxxxxxxxxxx 20 hours ago 156MB -malcolmnetsec/zeek 2.2.1 xxxxxxxxxxxx 20 hours ago 442MB +malcolmnetsec/curator 2.3.0 xxxxxxxxxxxx 40 hours ago 256MB +malcolmnetsec/elastalert 2.3.0 xxxxxxxxxxxx 40 hours ago 410MB +malcolmnetsec/elasticsearch-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 2.3.0 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 2.3.0 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 2.3.0 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/moloch 2.3.0 xxxxxxxxxxxx 17 hours ago 683MB +malcolmnetsec/name-map-ui 2.3.0 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 2.3.0 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 2.3.0 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 2.3.0 xxxxxxxxxxxx 39 hours ago 887MB ``` #### Import from pre-packaged tarballs @@ -217,6 +217,7 @@ Malcolm leverages the following excellent open source tools, among others. * [Logstash](https://www.elastic.co/products/logstash) and [Filebeat](https://www.elastic.co/products/beats/filebeat) - for ingesting and parsing [Zeek](https://www.zeek.org/index.html) [Log Files](https://docs.zeek.org/en/stable/script-reference/log-files.html) and ingesting them into Elasticsearch in a format that Moloch understands and is able to understand in the same way it natively understands PCAP data * [Kibana](https://www.elastic.co/products/kibana) - for creating additional ad-hoc visualizations and dashboards beyond that which is provided by Moloch Viewer * [Zeek](https://www.zeek.org/index.html) - a network analysis framework and IDS +* [Yara](https://github.com/VirusTotal/yara) - a tool used to identify and classify malware samples * [ClamAV](https://www.clamav.net/) - an antivirus engine for scanning files extracted by Zeek * [CyberChef](https://github.com/gchq/CyberChef) - a "swiss-army knife" data conversion tool * [jQuery File Upload](https://github.com/blueimp/jQuery-File-Upload) - for uploading PCAP files and Zeek logs for processing @@ -225,7 +226,8 @@ Malcolm leverages the following excellent open source tools, among others. * [Nginx](https://nginx.org/) - for HTTPS and reverse proxying Malcolm components * [nginx-auth-ldap](https://github.com/kvspb/nginx-auth-ldap) - an LDAP authentication module for nginx * [ElastAlert](https://github.com/Yelp/elastalert) - an alerting framework for Elasticsearch. Specifically, the [BitSensor fork of ElastAlert](https://github.com/bitsensor/elastalert), its Docker configuration and its corresponding [Kibana plugin](https://github.com/bitsensor/elastalert-kibana-plugin) are used. -* [freq](https://github.com/MarkBaggett/freq) - a tool for calculating entropy of strings +* [Mark Baggett](https://github.com/MarkBaggett)'s [freq](https://github.com/MarkBaggett/freq) - a tool for calculating entropy of strings +* [Florian Roth](https://github.com/Neo23x0)'s [Signature-Base](https://github.com/Neo23x0/signature-base) Yara ruleset * These Zeek plugins: * Amazon.com, Inc.'s [ICS protocol](https://github.com/amzn?q=zeek) analyzers * Andrew Klaus's [Sniffpass](https://github.com/cybera/zeek-sniffpass) plugin for detecting cleartext passwords in HTTP POST requests @@ -517,7 +519,11 @@ Various other environment variables inside of `docker-compose.yml` can be tweake * `VTOT_API2_KEY` – used to specify a [VirusTotal Public API v.20](https://www.virustotal.com/en/documentation/public-api/) key, which, if specified, will be used to submit hashes of [Zeek-extracted files](#ZeekFileExtraction) to VirusTotal -* `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true` (and `VTOT_API2_KEY` is unspecified), [Zeek-extracted files](#ZeekFileExtraction) will be scanned with ClamAV +* `EXTRACTED_FILE_ENABLE_YARA` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with Yara + +* `EXTRACTED_FILE_YARA_CUSTOM_ONLY` – if set to `true`, Malcolm will bypass the default [Yara ruleset](https://github.com/Neo23x0/signature-base) and use only user-defined rules in `./yara/rules` + +* `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](#ZeekFileExtraction) will be scanned with ClamAV * `EXTRACTED_FILE_ENABLE_FRESHCLAM` – if set to `true`, ClamAV will periodically update virus databases @@ -1260,10 +1266,11 @@ To specify which files should be extracted, the following values are acceptable * `known`: extraction of files for which any mime type can be determined * `all`: extract all files -Extracted files can be examined through either (but not both) of two methods: +Extracted files can be examined through any of the following methods: * submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, specify the `VTOT_API2_KEY` [environment variable in `docker-compose.yml`](#DockerComposeYml) -* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, set the `EXTRACTED_FILE_ENABLE_CLAMAV` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true` and leave `VTOT_API2_KEY` blank +* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, set the `EXTRACTED_FILE_ENABLE_CLAMAV` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true` +* scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, set the `EXTRACTED_FILE_ENABLE_YARA` [environment variable in `docker-compose.yml`](#DockerComposeYml) to `true` Files which are flagged as potentially malicious via either of these methods will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in Kibana. @@ -1415,7 +1422,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-2.2.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-2.3.0.iso" … ``` @@ -1773,6 +1780,10 @@ Scan extracted files with ClamAV? (y/N): y Download updated ClamAV virus signatures periodically? (Y/n): y +Scan extracted files with Yara? (y/N): y + +Lookup extracted file hashes with VirusTotal? (y/N): n + Should Malcolm capture network traffic to PCAP files? (y/N): y Specify capture interface(s) (comma-separated): eth0 @@ -1797,11 +1808,11 @@ Pulling elasticsearch ... done Pulling file-monitor ... done Pulling filebeat ... done Pulling freq ... done -Pulling name-map-ui ... done Pulling htadmin ... done Pulling kibana ... done Pulling logstash ... done Pulling moloch ... done +Pulling name-map-ui ... done Pulling nginx-proxy ... done Pulling pcap-capture ... done Pulling pcap-monitor ... done @@ -1810,22 +1821,22 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/curator 2.2.1 xxxxxxxxxxxx 20 hours ago 246MB -malcolmnetsec/elastalert 2.2.1 xxxxxxxxxxxx 20 hours ago 408MB -malcolmnetsec/elasticsearch-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 693MB -malcolmnetsec/filebeat-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 474MB -malcolmnetsec/file-monitor 2.2.1 xxxxxxxxxxxx 20 hours ago 386MB -malcolmnetsec/file-upload 2.2.1 xxxxxxxxxxxx 20 hours ago 199MB -malcolmnetsec/freq 2.2.1 xxxxxxxxxxxx 20 hours ago 390MB -malcolmnetsec/htadmin 2.2.1 xxxxxxxxxxxx 20 hours ago 180MB -malcolmnetsec/kibana-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 1.07GB -malcolmnetsec/logstash-oss 2.2.1 xxxxxxxxxxxx 20 hours ago 1.05GB -malcolmnetsec/moloch 2.2.1 xxxxxxxxxxxx 20 hours ago 667MB -malcolmnetsec/name-map-ui 2.2.1 xxxxxxxxxxxx 20 hours ago 134MB -malcolmnetsec/nginx-proxy 2.2.1 xxxxxxxxxxxx 20 hours ago 118MB -malcolmnetsec/pcap-capture 2.2.1 xxxxxxxxxxxx 20 hours ago 111MB -malcolmnetsec/pcap-monitor 2.2.1 xxxxxxxxxxxx 20 hours ago 156MB -malcolmnetsec/zeek 2.2.1 xxxxxxxxxxxx 20 hours ago 442MB +malcolmnetsec/curator 2.3.0 xxxxxxxxxxxx 40 hours ago 256MB +malcolmnetsec/elastalert 2.3.0 xxxxxxxxxxxx 40 hours ago 410MB +malcolmnetsec/elasticsearch-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 2.3.0 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 2.3.0 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 2.3.0 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-oss 2.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 2.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/moloch 2.3.0 xxxxxxxxxxxx 17 hours ago 683MB +malcolmnetsec/name-map-ui 2.3.0 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 2.3.0 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 2.3.0 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 2.3.0 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 2.3.0 xxxxxxxxxxxx 39 hours ago 887MB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 652161610..a2f4bceaf 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -47,6 +47,8 @@ x-zeek-variables: &zeek-variables EXTRACTED_FILE_MAX_BYTES : 134217728 VTOT_API2_KEY : '0' VTOT_REQUESTS_PER_MINUTE : 4 + EXTRACTED_FILE_ENABLE_YARA : 'false' + EXTRACTED_FILE_YARA_CUSTOM_ONLY : 'false' EXTRACTED_FILE_ENABLE_CLAMAV : 'false' EXTRACTED_FILE_ENABLE_FRESHCLAM : 'false' EXTRACTED_FILE_PIPELINE_DEBUG : 'false' @@ -122,7 +124,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: elasticsearch: - image: malcolmnetsec/elasticsearch-oss:2.2.1 + image: malcolmnetsec/elasticsearch-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -157,7 +159,7 @@ services: retries: 3 start_period: 180s kibana: - image: malcolmnetsec/kibana-oss:2.2.1 + image: malcolmnetsec/kibana-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -183,7 +185,7 @@ services: retries: 3 start_period: 210s elastalert: - image: malcolmnetsec/elastalert:2.2.1 + image: malcolmnetsec/elastalert:2.3.0 restart: "no" stdin_open: false tty: true @@ -211,7 +213,7 @@ services: retries: 3 start_period: 210s curator: - image: malcolmnetsec/curator:2.2.1 + image: malcolmnetsec/curator:2.3.0 restart: "no" stdin_open: false tty: true @@ -230,7 +232,7 @@ services: retries: 3 start_period: 30s logstash: - image: malcolmnetsec/logstash-oss:2.2.1 + image: malcolmnetsec/logstash-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -263,7 +265,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:2.2.1 + image: malcolmnetsec/filebeat-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -300,7 +302,7 @@ services: retries: 3 start_period: 60s moloch: - image: malcolmnetsec/moloch:2.2.1 + image: malcolmnetsec/moloch:2.3.0 restart: "no" stdin_open: false tty: true @@ -311,7 +313,7 @@ services: << : *process-variables << : *common-upload-variables << : *moloch-variables - MOLOCH_VERSION : '2.3.2' + MOLOCH_VERSION : '2.4.0' VIRTUAL_HOST : 'moloch.malcolm.local' ES_HOST : 'elasticsearch' ES_PORT : 9200 @@ -339,7 +341,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:2.2.1 + image: malcolmnetsec/zeek:2.3.0 restart: "no" stdin_open: false tty: true @@ -365,7 +367,7 @@ services: retries: 3 start_period: 60s file-monitor: - image: malcolmnetsec/file-monitor:2.2.1 + image: malcolmnetsec/file-monitor:2.3.0 restart: "no" stdin_open: false tty: true @@ -378,14 +380,15 @@ services: volumes: - ./zeek-logs/extract_files:/data/zeek/extract_files - ./zeek-logs/current:/data/zeek/logs + - ./yara/rules:/yara-rules/custom:ro healthcheck: - test: ["CMD", "supervisorctl", "status", "watcher", "scanner", "logger"] + test: ["CMD", "supervisorctl", "status", "watcher", "logger"] interval: 30s timeout: 15s retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:2.2.1 + image: malcolmnetsec/pcap-capture:2.3.0 restart: "no" stdin_open: false tty: true @@ -411,7 +414,7 @@ services: retries: 3 start_period: 60s pcap-monitor: - image: malcolmnetsec/pcap-monitor:2.2.1 + image: malcolmnetsec/pcap-monitor:2.3.0 restart: "no" stdin_open: false tty: true @@ -434,7 +437,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:2.2.1 + image: malcolmnetsec/file-upload:2.3.0 restart: "no" stdin_open: false tty: true @@ -460,7 +463,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:2.2.1 + image: malcolmnetsec/htadmin:2.3.0 restart: "no" stdin_open: false tty: true @@ -482,7 +485,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:2.2.1 + image: malcolmnetsec/freq:2.3.0 restart: "no" stdin_open: false tty: true @@ -500,7 +503,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:2.2.1 + image: malcolmnetsec/name-map-ui:2.3.0 restart: "no" stdin_open: false tty: true @@ -521,7 +524,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:2.2.1 + image: malcolmnetsec/nginx-proxy:2.3.0 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index 0a0380632..571cd9505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,8 @@ x-zeek-variables: &zeek-variables EXTRACTED_FILE_MAX_BYTES : 134217728 VTOT_API2_KEY : '0' VTOT_REQUESTS_PER_MINUTE : 4 + EXTRACTED_FILE_ENABLE_YARA : 'false' + EXTRACTED_FILE_YARA_CUSTOM_ONLY : 'false' EXTRACTED_FILE_ENABLE_CLAMAV : 'false' EXTRACTED_FILE_ENABLE_FRESHCLAM : 'false' EXTRACTED_FILE_PIPELINE_DEBUG : 'false' @@ -125,7 +127,7 @@ services: build: context: . dockerfile: Dockerfiles/elasticsearch.Dockerfile - image: malcolmnetsec/elasticsearch-oss:2.2.1 + image: malcolmnetsec/elasticsearch-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -163,7 +165,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana.Dockerfile - image: malcolmnetsec/kibana-oss:2.2.1 + image: malcolmnetsec/kibana-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -192,7 +194,7 @@ services: build: context: . dockerfile: Dockerfiles/elastalert.Dockerfile - image: malcolmnetsec/elastalert:2.2.1 + image: malcolmnetsec/elastalert:2.3.0 restart: "no" stdin_open: false tty: true @@ -223,7 +225,7 @@ services: build: context: . dockerfile: Dockerfiles/curator.Dockerfile - image: malcolmnetsec/curator:2.2.1 + image: malcolmnetsec/curator:2.3.0 restart: "no" stdin_open: false tty: true @@ -247,7 +249,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:2.2.1 + image: malcolmnetsec/logstash-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -285,7 +287,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:2.2.1 + image: malcolmnetsec/filebeat-oss:2.3.0 restart: "no" stdin_open: false tty: true @@ -326,7 +328,7 @@ services: build: context: . dockerfile: Dockerfiles/moloch.Dockerfile - image: malcolmnetsec/moloch:2.2.1 + image: malcolmnetsec/moloch:2.3.0 restart: "no" stdin_open: false tty: true @@ -337,7 +339,7 @@ services: << : *process-variables << : *common-upload-variables << : *moloch-variables - MOLOCH_VERSION : '2.3.2' + MOLOCH_VERSION : '2.4.0' VIRTUAL_HOST : 'moloch.malcolm.local' ES_HOST : 'elasticsearch' ES_PORT : 9200 @@ -371,7 +373,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:2.2.1 + image: malcolmnetsec/zeek:2.3.0 restart: "no" stdin_open: false tty: true @@ -401,7 +403,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:2.2.1 + image: malcolmnetsec/file-monitor:2.3.0 restart: "no" stdin_open: false tty: true @@ -414,8 +416,9 @@ services: volumes: - ./zeek-logs/extract_files:/data/zeek/extract_files - ./zeek-logs/current:/data/zeek/logs + - ./yara/rules:/yara-rules/custom:ro healthcheck: - test: ["CMD", "supervisorctl", "status", "watcher", "scanner", "logger"] + test: ["CMD", "supervisorctl", "status", "watcher", "logger"] interval: 30s timeout: 15s retries: 3 @@ -424,7 +427,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:2.2.1 + image: malcolmnetsec/pcap-capture:2.3.0 restart: "no" stdin_open: false tty: true @@ -453,7 +456,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:2.2.1 + image: malcolmnetsec/pcap-monitor:2.3.0 restart: "no" stdin_open: false tty: true @@ -479,7 +482,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:2.2.1 + image: malcolmnetsec/file-upload:2.3.0 restart: "no" stdin_open: false tty: true @@ -505,7 +508,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:2.2.1 + image: malcolmnetsec/htadmin:2.3.0 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -530,7 +533,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:2.2.1 + image: malcolmnetsec/freq:2.3.0 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -551,7 +554,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:2.2.1 + image: malcolmnetsec/name-map-ui:2.3.0 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -578,7 +581,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:2.2.1 + image: malcolmnetsec/nginx-proxy:2.3.0 restart: "no" stdin_open: false tty: true diff --git a/file-monitor/docker-entrypoint.sh b/file-monitor/docker-entrypoint.sh new file mode 100755 index 000000000..f4a7e1f8e --- /dev/null +++ b/file-monitor/docker-entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved. + +if [[ -z $EXTRACTED_FILE_ENABLE_CLAMAV ]]; then + EXTRACTED_FILE_ENABLE_CLAMAV=false +fi + +if [[ -z $EXTRACTED_FILE_ENABLE_YARA ]]; then + EXTRACTED_FILE_ENABLE_YARA=false +fi + +if [[ -z $EXTRACTED_FILE_ENABLE_MALASS ]]; then + [[ ${#MALASS_HOST} -gt 1 ]] && EXTRACTED_FILE_ENABLE_MALASS=true || EXTRACTED_FILE_ENABLE_MALASS=false +fi + +if [[ -z $EXTRACTED_FILE_ENABLE_VTOT ]]; then + [[ ${#VTOT_API2_KEY} -gt 1 ]] && EXTRACTED_FILE_ENABLE_VTOT=true || EXTRACTED_FILE_ENABLE_VTOT=false +fi + +export EXTRACTED_FILE_ENABLE_CLAMAV +export EXTRACTED_FILE_ENABLE_YARA +export EXTRACTED_FILE_ENABLE_MALASS +export EXTRACTED_FILE_ENABLE_VTOT + +exec "$@" diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 2d12a0763..df82c688b 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -35,19 +35,69 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true -[program:scanner] -command=/usr/local/bin/zeek_carve_scanner.py +[group:scanners] +programs=virustotal,clamav,yara,malass + +[program:virustotal] +command=/usr/local/bin/vtot_scan.py --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s --vtot-api %(ENV_VTOT_API2_KEY)s --vtot-req-limit %(ENV_VTOT_REQUESTS_PER_MINUTE)s +autostart=%(ENV_EXTRACTED_FILE_ENABLE_VTOT)s +startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s +startretries=0 +stopasgroup=true +killasgroup=true +directory=/data/zeek/extract_files +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:clamav] +command=/usr/local/bin/clam_scan.py + --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s + --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s + --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s + --clamav %(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s + --clamav-socket "%(ENV_CLAMD_SOCKET_FILE)s" +autostart=%(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s +startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s +startretries=0 +stopasgroup=true +killasgroup=true +directory=/data/zeek/extract_files +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:yara] +command=/usr/local/bin/yara_scan.py + --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s + --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s + --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s + --yara %(ENV_EXTRACTED_FILE_ENABLE_YARA)s + --yara-custom-only %(ENV_EXTRACTED_FILE_YARA_CUSTOM_ONLY)s +autostart=%(ENV_EXTRACTED_FILE_ENABLE_YARA)s +startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s +startretries=0 +stopasgroup=true +killasgroup=true +directory=/data/zeek/extract_files +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:malass] +command=/usr/local/bin/malass_scan.py + --verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG)s + --extra-verbose %(ENV_EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA)s + --start-sleep %(ENV_EXTRACTED_FILE_SCANNER_START_SLEEP)s --malass-host "%(ENV_MALASS_HOST)s" --malass-port %(ENV_MALASS_PORT)s --malass-limit %(ENV_MALASS_MAX_REQUESTS)s - --clamav %(ENV_EXTRACTED_FILE_ENABLE_CLAMAV)s - --clamav-socket "%(ENV_CLAMD_SOCKET_FILE)s" -autostart=true +autostart=%(ENV_EXTRACTED_FILE_ENABLE_MALASS)s startsecs=%(ENV_EXTRACTED_FILE_WATCHER_START_SLEEP)s startretries=0 stopasgroup=true diff --git a/logstash/pipelines/zeek/11_zeek_logs.conf b/logstash/pipelines/zeek/11_zeek_logs.conf index 1be262e7a..45d8354db 100644 --- a/logstash/pipelines/zeek/11_zeek_logs.conf +++ b/logstash/pipelines/zeek/11_zeek_logs.conf @@ -3347,14 +3347,17 @@ filter { code => " matchesHash = Hash.new idArray = Array.new - engineArray = Array.new event.get('[zeek_signatures][event_message]').split(';').each { |hit| nameAndEngines = hit.split(/(.+?)\s*<(.+)>/) nameAndEngines[2].split(',').each { |engine| - matchesHash[engine] = nameAndEngines[1] + unless matchesHash.key?(engine) + matchesHash[engine] = Array.new + end + matchesHash[engine].push(nameAndEngines[1]) + idArray.push(nameAndEngines[1]) } } - event.set('[zeek_signatures][signature_id]', matchesHash.values.uniq) + event.set('[zeek_signatures][signature_id]', idArray.uniq) event.set('[zeek_signatures][engine]', matchesHash.keys) event.set('[zeek_signatures][hits]', matchesHash)" } diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index bc71d4838..144838a60 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -74,12 +74,21 @@ if [ -d "$WORKDIR" ]; then echo "linux-image-$(uname -r)" > ./config/package-lists/kernel.list.chroot echo "linux-headers-$(uname -r)" >> ./config/package-lists/kernel.list.chroot echo "linux-compiler-gcc-8-x86=$(dpkg -s linux-compiler-gcc-8-x86 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot - echo "linux-kbuild-5.6=$(dpkg -s linux-kbuild-5.6 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot + echo "linux-kbuild-5.7=$(dpkg -s linux-kbuild-5.7 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-linux=$(dpkg -s firmware-linux | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-linux-nonfree=$(dpkg -s firmware-linux-nonfree | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-misc-nonfree=$(dpkg -s firmware-misc-nonfree | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-amd-graphics=$(dpkg -s firmware-amd-graphics | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot + # and make sure we remove the old stuff when it's all over + echo "#!/bin/sh" > ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "export LC_ALL=C.UTF-8" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "export LANG=C.UTF-8" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get -y --purge remove *4.19* || true" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get -y autoremove" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get clean" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + chmod +x ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + # grab things from the Malcolm parent directory into /etc/skel so the user's got it set up in their home/Malcolm dir pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 MALCOLM_DEST_DIR="$WORKDIR/work/$IMAGE_NAME-Live-Build/config/includes.chroot/etc/skel/Malcolm" @@ -98,9 +107,11 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/" mkdir -p "$MALCOLM_DEST_DIR/pcap/processed/" mkdir -p "$MALCOLM_DEST_DIR/scripts/" + mkdir -p "$MALCOLM_DEST_DIR/yara/rules/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/current/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/" + mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/extract_files/" YML_IMAGE_VERSION="$(grep -P "^\s+image:\s*malcolm" ./docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && IMAGE_VERSION="$YML_IMAGE_VERSION" cp ./docker-compose-standalone.yml "$MALCOLM_DEST_DIR/docker-compose.yml" diff --git a/moloch/patch/footer_links.patch b/moloch/patch/footer_links.patch new file mode 100644 index 000000000..1b28516e6 --- /dev/null +++ b/moloch/patch/footer_links.patch @@ -0,0 +1,14 @@ +diff --git a/viewer/vueapp/src/components/utils/Footer.vue b/viewer/vueapp/src/components/utils/Footer.vue +index 55b2dbdb..8e6338da 100644 +--- a/viewer/vueapp/src/components/utils/Footer.vue ++++ b/viewer/vueapp/src/components/utils/Footer.vue +@@ -4,7 +4,8 @@ +

+ + Moloch v{{molochVersion}} | +- molo.ch ++ molo.ch | ++ Malcolm README + + | {{ responseTime | commaString }}ms + diff --git a/moloch/patch/help_links.patch b/moloch/patch/help_links.patch deleted file mode 100644 index 3334de4f1..000000000 --- a/moloch/patch/help_links.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/viewer/vueapp/src/components/help/Help.vue b/viewer/vueapp/src/components/help/Help.vue -index ba43003a..89b292f0 100644 ---- a/viewer/vueapp/src/components/help/Help.vue -+++ b/viewer/vueapp/src/components/help/Help.vue -@@ -148,6 +148,9 @@ - -

-
-+ Malcolm on GitHub | -+ Malcolm README | -+ CyberChef | - Home Page | - FAQ | - Wiki | diff --git a/scripts/control.py b/scripts/control.py index cd7f76683..ac0a181f9 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -98,7 +98,7 @@ def logs(): # increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output osEnv = os.environ.copy() - osEnv['COMPOSE_HTTP_TIMEOUT'] = '999999999' + osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000' process = Popen([dockerComposeBin, '-f', args.composeFile, 'logs', '-f'], env=osEnv, stdout=PIPE) while True: @@ -268,7 +268,7 @@ def start(): # increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output osEnv = os.environ.copy() - osEnv['COMPOSE_HTTP_TIMEOUT'] = '999999999' + osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000' # start docker err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'up', '--detach'], env=osEnv, debug=args.debug) diff --git a/scripts/install.py b/scripts/install.py index 0518e7eb3..b7bde9dcb 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -372,6 +372,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa fileCarveMode = None filePreserveMode = None vtotApiKey = '0' + yaraScan = False clamAvScan = False clamAvUpdate = False @@ -384,7 +385,9 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa if InstallerYesOrNo('Scan extracted files with ClamAV?', default=False): clamAvScan = True clamAvUpdate = InstallerYesOrNo('Download updated ClamAV virus signatures periodically?', default=True) - elif InstallerYesOrNo('Lookup extracted file hashes with VirusTotal?', default=False): + if InstallerYesOrNo('Scan extracted files with Yara?', default=False): + yaraScan = True + if InstallerYesOrNo('Lookup extracted file hashes with VirusTotal?', default=False): while (len(vtotApiKey) <= 1): vtotApiKey = InstallerAskForString('Enter VirusTotal API key') @@ -467,6 +470,9 @@ def tweak_malcolm_runtime(self, malcolm_install_path, expose_logstash_default=Fa elif 'VTOT_API2_KEY' in line: # virustotal API key line = re.sub(r'(VTOT_API2_KEY\s*:\s*)(\S+)', r"\g<1>'{}'".format(vtotApiKey), line) + elif 'EXTRACTED_FILE_ENABLE_YARA' in line: + # file scanning via yara + line = re.sub(r'(EXTRACTED_FILE_ENABLE_YARA\s*:\s*)(\S+)', r'\g<1>{}'.format("'true'" if yaraScan else "'false'"), line) elif 'EXTRACTED_FILE_ENABLE_CLAMAV' in line: # file scanning via clamav line = re.sub(r'(EXTRACTED_FILE_ENABLE_CLAMAV\s*:\s*)(\S+)', r'\g<1>{}'.format("'true'" if clamAvScan else "'false'"), line) diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index e422e7213..3dc3ee5e2 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -75,6 +75,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/moloch-logs/" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/" mkdir $VERBOSE -p "$DESTDIR/pcap/processed/" + mkdir $VERBOSE -p "$DESTDIR/yara/rules/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/current/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/upload/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/processed/" diff --git a/sensor-iso/README.md b/sensor-iso/README.md index d05b01171..ea3408d65 100644 --- a/sensor-iso/README.md +++ b/sensor-iso/README.md @@ -200,14 +200,17 @@ To specify which files should be extracted, specify the Zeek file carving mode: If you're not sure what to choose, either of **mapped (except common plain text files)** (if you want to carve and scan almost all files) or **interesting** (if you only want to carve and scan files with [mime types of common attack vectors](./interface/sensor_ctl/extractor_override.interesting.zeek)) is probably a good choice. -Extracted files can be examined through either (but not both) of two methods: +Next, specify which carved files to preserve (saved on the sensor under `/capture/bro/capture/extract_files/quarantine` by default). In order to not consume all of the sensor's available storage space, the oldest preserved files will be pruned along with the oldest Zeek logs as described below with **AUTOSTART_PRUNE_ZEEK** in the [autostart services](#ConfigAutostart) section. -* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, enable **AUTOSTART_CLAMAV_SERVICE** when configuring [autostart services](#ConfigAutostart) and leave `VTOT_API2_KEY` value (described below) blank -* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method manually edit `/opt/sensor/sensor_ctl/control_vars.conf` and specify your [VirusTotal API key](https://developers.virustotal.com/reference) in `VTOT_API2_KEY` +You'll be prompted to specify which engine(s) to use to analyze extracted files. Extracted files can be examined through any of three methods: -Files which are flagged as potentially malicious via either of these methods will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [Kibana](https://github.com/idaholab/malcolm#KibanaVisualizations) when forwarded to Malcolm. +![File scanners](./docs/images/zeek_file_carve_scanners.png) -Next, specify which carved files to preserve (saved on the sensor under `/capture/bro/capture/extract_files/quarantine` by default). In order to not consume all of the sensor's available storage space, the oldest preserved files will be pruned along with the oldest Zeek logs as described below with **AUTOSTART_PRUNE_ZEEK** in the [autostart services](#ConfigAutostart) section. +* scanning files with [**ClamAV**](https://www.clamav.net/); to enable this method, select **ZEEK_FILE_SCAN_CLAMAV** when specifying scanners for Zeek-carved files +* submitting file hashes to [**VirusTotal**](https://www.virustotal.com/en/#search); to enable this method, select **ZEEK_FILE_SCAN_VTOT** when specifying scanners for Zeek-carved files, then manually edit `/opt/sensor/sensor_ctl/control_vars.conf` and specify your [VirusTotal API key](https://developers.virustotal.com/reference) in `VTOT_API2_KEY` +* scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, select **ZEEK_FILE_SCAN_YARA** when specifying scanners for Zeek-carved files + +Files which are flagged as potentially malicious will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [Kibana](https://github.com/idaholab/malcolm#KibanaVisualizations) when forwarded to Malcolm. ![File quarantine](./docs/images/file_quarantine.png) @@ -328,7 +331,6 @@ Once the forwarders have been configured, the final step is to **Configure Autos Despite configuring capture and/or forwarder services as described in previous sections, only services enabled in the autostart configuration will run when the sensor starts up. The available autostart processes are as follows (recommended services are in **bold text**): * **AUTOSTART_AUDITBEAT** – [auditbeat](#auditbeat) audit log forwarder -* **AUTOSTART_CLAMAV_SERVICE** – ClamAV service for scanning [files carved from traffic streams by Zeek](#ZeekFileExtraction) for trojans, viruses, malware and other malicious threats * **AUTOSTART_CLAMAV_UPDATES** – Virus database update service for ClamAV (requires sensor to be connected to the internet) * **AUTOSTART_FILEBEAT** – [filebeat](#filebeat) Zeek log forwarder * **AUTOSTART_HEATBEAT** – [sensor hardware](#heatbeat) (eg., CPU and storage device temperature) metrics forwarder @@ -341,7 +343,6 @@ Despite configuring capture and/or forwarder services as described in previous s * **AUTOSTART_SYSLOGBEAT** – filebeat [system log forwarder](#syslogbeat) * *AUTOSTART_TCPDUMP* – [tcpdump](https://www.tcpdump.org/) PCAP engine for saving packet capture (PCAP) files * **AUTOSTART_ZEEK** – [Zeek](https://www.zeek.org/) traffic analysis engine -* **AUTOSTART_ZEEK_FILE_WATCH** – monitor for [files carved from traffic streams by Zeek](#ZeekFileExtraction) Note that only one packet capture engine ([moloch-capture](https://molo.ch/), [netsniff-ng](http://netsniff-ng.org/), or [tcpdump](https://www.tcpdump.org/)) can be used. @@ -376,7 +377,9 @@ prune:prune-pcap RUNNING pid 14446, uptime 8 days, 20:22:32 prune:prune-zeek RUNNING pid 14442, uptime 8 days, 20:22:32 tcpdump:tcpdump-enp8s0 STOPPED Not started zeek:logger RUNNING pid 14434, uptime 8 days, 20:22:32 -zeek:scanner RUNNING pid 14435, uptime 8 days, 20:22:32 +zeek:virustotal RUNNING pid 14435, uptime 8 days, 20:22:32 +zeek:yara RUNNING pid 14435, uptime 8 days, 20:22:32 +zeek:clamav RUNNING pid 14435, uptime 8 days, 20:22:32 zeek:watcher RUNNING pid 14441, uptime 8 days, 20:22:32 zeek:zeekctl RUNNING pid 14433, uptime 8 days, 20:22:32 ``` @@ -399,7 +402,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-2.2.1.iso" +Finished, created "/sensor-build/hedgehog-2.3.0.iso" … ``` diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index bb6072aff..812f05d8a 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -77,12 +77,21 @@ if [ -d "$WORKDIR" ]; then echo "linux-image-$(uname -r)" > ./config/package-lists/kernel.list.chroot echo "linux-headers-$(uname -r)" >> ./config/package-lists/kernel.list.chroot echo "linux-compiler-gcc-8-x86=$(dpkg -s linux-compiler-gcc-8-x86 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot - echo "linux-kbuild-5.6=$(dpkg -s linux-kbuild-5.6 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot + echo "linux-kbuild-5.7=$(dpkg -s linux-kbuild-5.7 | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-linux=$(dpkg -s firmware-linux | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-linux-nonfree=$(dpkg -s firmware-linux-nonfree | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-misc-nonfree=$(dpkg -s firmware-misc-nonfree | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot echo "firmware-amd-graphics=$(dpkg -s firmware-amd-graphics | grep ^Version: | cut -d' ' -f2)" >> ./config/package-lists/kernel.list.chroot + # and make sure we remove the old stuff when it's all over + echo "#!/bin/sh" > ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "export LC_ALL=C.UTF-8" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "export LANG=C.UTF-8" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get -y --purge remove *4.19* || true" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get -y autoremove" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + echo "apt-get clean" >> ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + chmod +x ./config/hooks/normal/9999-remove-old-kernel-artifacts.hook.chroot + mkdir -p ./config/includes.chroot/opt/hedgehog_install_artifacts # copy the interface code into place for the resultant image diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot index b554c6d7c..36f87c0df 100755 --- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -21,4 +21,5 @@ pip3 install --no-compile --no-cache-dir --force-reinstall --upgrade \ pyzmq \ requests \ scapy \ + yara-python \ zkg diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 2f1c2eeb9..008e6669c 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -25,6 +25,11 @@ CMAKE_URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmak BISON_VER="3.6.2" BISON_URL="https://ftp.gnu.org/gnu/bison/bison-${BISON_VER}.tar.gz" +YARA_VERSION="4.0.2" +YARA_URL="https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz" +YARA_RULES_URL="https://codeload.github.com/Neo23x0/signature-base/tar.gz/master" +YARA_RULES_DIR="/opt/yara-rules" + mkdir -p /opt/hedgehog_install_artifacts/ # some environment variables needed for build using clang @@ -118,6 +123,37 @@ mv ./zeek-$ZEEK_VER-hedgehog.tar.gz /opt/hedgehog_install_artifacts/ rm -Rf zeek-$ZEEK_VER* ### +# yara +mkdir -p usr/local/src +cd /usr/local/src + +curl -sSL "${YARA_URL}" | tar xzf - -C /usr/local/src/ +cd "./yara-${YARA_VERSION}" +./bootstrap.sh +./configure --prefix=/usr + --with-crypto + --enable-magic + --enable-cuckoo + --enable-dotnet +make +#make install +checkinstall -y -D --strip=yes --stripso=yes --install=yes --fstrans=no --pkgname="yara" --pkgversion="$YARA_VERSION" --pkgarch="amd64" --pkgsource="$YARA_URL" +cp *.deb /opt/hedgehog_install_artifacts/ +cd /tmp +rm -rf /usr/local/src/yara* + +mkdir -p ./Neo23x0 +curl -sSL "$YARA_RULES_URL" | tar xzvf - -C ./Neo23x0 --strip-components 1 +mkdir -p "${YARA_RULES_DIR}"/custom +cp ./Neo23x0/yara/* ./Neo23x0/vendor/yara/* "${YARA_RULES_DIR}"/ +cp ./Neo23x0/LICENSE "${YARA_RULES_DIR}"/_LICENSE +rm -rf /tmp/Neo23x0 + +cd "${YARA_RULES_DIR}"/.. +tar czf yara-rules-hedgehog.tar.gz "$(basename "${YARA_RULES_DIR}")" +mv ./yara-rules-hedgehog.tar.gz /opt/hedgehog_install_artifacts/ +### + # update clamav signatures freshclam --stdout --quiet --no-warnings ### diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 1b43983e0..0b4810086 100644 --- a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -82,7 +82,7 @@ redef ignore_checksums = T; @if (!disable_wireguard) @load ./spicy-noise - event zeek_init() { + event zeek_init() &priority=-5 { if (disable_wireguard_transport_packets) { Log::remove_default_filter(WireGuard::WGLOG); Log::add_filter(WireGuard::WGLOG, diff --git a/sensor-iso/config/package-lists/system.list.chroot b/sensor-iso/config/package-lists/system.list.chroot index 439abc8fa..d168371d2 100644 --- a/sensor-iso/config/package-lists/system.list.chroot +++ b/sensor-iso/config/package-lists/system.list.chroot @@ -92,8 +92,12 @@ libgoogle-perftools-dev libgoogle-perftools4 libgtk2.0-bin libgtk2.0-dev +libjansson-dev +libjansson4 libjson-perl libkrb5-dev +libmagic-dev +libmagic1 libmaxminddb-dev libmaxminddb0 libnacl-dev @@ -111,7 +115,9 @@ libpcap0.8-dev librocksdb5.17 libsodium-dev libssl-dev +libssl1.1 libtcmalloc-minimal4 +libtool libunwind8 liburcu-dev libwww-perl @@ -150,6 +156,7 @@ patch pciutils pcregrep pigz +pkg-config pmount policykit-1 prelink diff --git a/sensor-iso/docs/Notes.md b/sensor-iso/docs/Notes.md index 837e57937..2ca89114b 100644 --- a/sensor-iso/docs/Notes.md +++ b/sensor-iso/docs/Notes.md @@ -113,12 +113,12 @@ $ /usr/sbin/tcpdump \ ### Compiling Moloch from source -At the time of writing, the [current stable release](https://github.com/aol/moloch/blob/master/CHANGELOG) of Moloch is [v2.3.2](https://github.com/aol/moloch/releases/tag/v2.3.2). The following bash script was used to install Moloch's build dependencies, download Moloch, build a Debian .deb package using [fpm](https://github.com/jordansissel/fpm) and install it. In building Hedgehog Linux, the building of this .deb is done inside a Docker container dedicated to that purpose. +At the time of writing, the [current stable release](https://github.com/aol/moloch/blob/master/CHANGELOG) of Moloch is [v2.4.0](https://github.com/aol/moloch/releases/tag/v2.4.0). The following bash script was used to install Moloch's build dependencies, download Moloch, build a Debian .deb package using [fpm](https://github.com/jordansissel/fpm) and install it. In building Hedgehog Linux, the building of this .deb is done inside a Docker container dedicated to that purpose. ```bash #!/bin/bash -MOLOCH_VERSION="2.3.2" +MOLOCH_VERSION="2.4.0" MOLOCHDIR="/opt/moloch" OUTPUT_DIR="/tmp" diff --git a/sensor-iso/docs/images/zeek_file_carve_scanners.png b/sensor-iso/docs/images/zeek_file_carve_scanners.png new file mode 100644 index 000000000..7be0b9a87 Binary files /dev/null and b/sensor-iso/docs/images/zeek_file_carve_scanners.png differ diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index 3e89139ac..25c8115b0 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -1,16 +1,16 @@ -certifi==2018.8.24 +certifi==2020.6.20 chardet==3.0.4 -click==6.7 -Flask==1.0.2 -Flask-Cors==3.0.6 -gunicorn==19.9.0 -idna==2.7 -itsdangerous==0.24 -Jinja2==2.10.1 -MarkupSafe==1.0 -psutil==5.6.6 -python-dotenv==0.9.1 -requests==2.20.0 -six==1.11.0 -urllib3==1.25.7 -Werkzeug==0.15.3 +click==7.1.2 +Flask==1.1.2 +Flask-Cors==3.0.8 +gunicorn==20.0.4 +idna==2.10 +itsdangerous==1.1.0 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +psutil==5.7.2 +python-dotenv==0.14.0 +requests==2.24.0 +six==1.15.0 +urllib3==1.25.10 +Werkzeug==1.0.1 diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 359dff1ec..f8aa3b278 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -10,6 +10,13 @@ export PCAP_SNAPLEN=0 export PCAP_MAX_DISK_FILL=90 export PCAP_PRUNE_CHECK_SECONDS=60 +export MOLOCH_VIEWER_PORT=8005 +export MOLOCH_PACKET_THREADS=5 +export MOLOCH_PACKET_ACL= + +export PROTOLOGBEAT_PORT=9515 +export PROTOLOGBEAT_INTERVAL=10 + export ZEEK_LOG_PATH=/home/sensor/bro_logs export ZEEK_MAX_DISK_FILL=90 export ZEEK_PRUNE_CHECK_SECONDS=90 @@ -22,14 +29,18 @@ export ZEEK_EXTRACTOR_OVERRIDE_FILE= export EXTRACTED_FILE_MIN_BYTES=64 export EXTRACTED_FILE_MAX_BYTES=134217728 export EXTRACTED_FILE_PRESERVATION=quarantined -export VTOT_API2_KEY="" -export MALASS_HOST="" -export MALASS_PORT=80 -export MOLOCH_VIEWER_PORT=8005 -export MOLOCH_PACKET_THREADS=5 -export MOLOCH_PACKET_ACL= -export PROTOLOGBEAT_PORT=9515 -export PROTOLOGBEAT_INTERVAL=10 +export ZEEK_DISABLE_MITRE_BZAR= +export ZEEK_DISABLE_HASH_ALL_FILES= +export ZEEK_DISABLE_LOG_PASSWORDS= +export ZEEK_DISABLE_MODBUS_TRACKING= +export ZEEK_DISABLE_MQTT= +export ZEEK_DISABLE_PE_XOR= +export ZEEK_DISABLE_QUIC= +export ZEEK_DISABLE_SSL_VALIDATE_CERTS= +export ZEEK_DISABLE_TELNET= +export ZEEK_DISABLE_TRACK_ALL_ASSETS= +export ZEEK_DISABLE_WIREGUARD= +export ZEEK_DISABLE_WIREGUARD_TRANSPORT_PACKETS=true # affects Moloch only for now: beats values are stored in keystores per-beat export ES_PROTOCOL=https @@ -39,8 +50,19 @@ export ES_USERNAME=sensor export ES_PASSWORD=%70%61%73%73%77%6F%72%64 export ES_SSL_VERIFY=none +export VTOT_API2_KEY="" +export MALASS_HOST="" +export MALASS_PORT=80 +export EXTRACTED_FILE_YARA_CUSTOM_ONLY=false +export YARA_RULES_DIR=/opt/yara-rules + +export ZEEK_FILE_WATCH=false +export ZEEK_FILE_SCAN_CLAMAV=false +export ZEEK_FILE_SCAN_VTOT=false +export ZEEK_FILE_SCAN_MALASS=false +export ZEEK_FILE_SCAN_YARA=false + export AUTOSTART_AUDITBEAT=false -export AUTOSTART_CLAMAV_SERVICE=false export AUTOSTART_CLAMAV_UPDATES=false export AUTOSTART_FILEBEAT=false export AUTOSTART_HEATBEAT=false @@ -53,4 +75,3 @@ export AUTOSTART_PRUNE_ZEEK=false export AUTOSTART_SYSLOGBEAT=false export AUTOSTART_TCPDUMP=false export AUTOSTART_ZEEK=false -export AUTOSTART_ZEEK_FILE_WATCH=false diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf index d75b96729..f4f6e6142 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf +++ b/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf @@ -14,7 +14,7 @@ killasgroup=true [program:clamav-service] command=/usr/sbin/clamd -c /etc/clamav/clamd.conf user=sensor -autostart=%(ENV_AUTOSTART_CLAMAV_SERVICE)s +autostart=%(ENV_ZEEK_FILE_SCAN_CLAMAV)s autorestart=true startsecs=0 startretries=0 diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf index 079ff4518..61773a440 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf +++ b/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf @@ -1,5 +1,5 @@ [group:zeek] -programs=zeekctl,watcher,scanner,logger +programs=zeekctl,watcher,virustotal,clamav,yara,malass,logger [program:zeekctl] command=/opt/zeek/bin/zeekdeploy.sh @@ -22,23 +22,58 @@ startsecs=100 startretries=3 stopasgroup=true killasgroup=true -autostart=%(ENV_AUTOSTART_ZEEK_FILE_WATCH)s +autostart=%(ENV_ZEEK_FILE_WATCH)s directory=%(ENV_ZEEK_LOG_PATH)s user=sensor -[program:scanner] +[program:virustotal] command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py --start-sleep 20 --vtot-api "%(ENV_VTOT_API2_KEY)s" +startsecs=30 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_ZEEK_FILE_SCAN_VTOT)s +directory=%(ENV_ZEEK_LOG_PATH)s +user=sensor + +[program:clamav] +command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py + --start-sleep 20 + --clamav %(ENV_ZEEK_FILE_SCAN_CLAMAV)s + --clamav-socket "%(ENV_SUPERVISOR_PATH)s/clamav/clamd.ctl" +startsecs=30 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_ZEEK_FILE_SCAN_CLAMAV)s +directory=%(ENV_ZEEK_LOG_PATH)s +user=sensor + +[program:yara] +command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py + --start-sleep 20 + --yara %(ENV_ZEEK_FILE_SCAN_YARA)s + --yara-custom-only "%(ENV_EXTRACTED_FILE_YARA_CUSTOM_ONLY)s" +startsecs=30 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_ZEEK_FILE_SCAN_YARA)s +directory=%(ENV_ZEEK_LOG_PATH)s +user=sensor + +[program:malass] +command=/usr/bin/python3.7 /usr/local/bin/zeek_carve_scanner.py + --start-sleep 20 --malass-host "%(ENV_MALASS_HOST)s" --malass-port %(ENV_MALASS_PORT)s - --clamav %(ENV_AUTOSTART_CLAMAV_SERVICE)s - --clamav-socket "%(ENV_SUPERVISOR_PATH)s/clamav/clamd.ctl" startsecs=30 startretries=3 stopasgroup=true killasgroup=true -autostart=%(ENV_AUTOSTART_ZEEK_FILE_WATCH)s +autostart=%(ENV_ZEEK_FILE_SCAN_MALASS)s directory=%(ENV_ZEEK_LOG_PATH)s user=sensor @@ -52,6 +87,6 @@ startsecs=20 startretries=3 stopasgroup=true killasgroup=true -autostart=%(ENV_AUTOSTART_ZEEK_FILE_WATCH)s +autostart=%(ENV_ZEEK_FILE_WATCH)s directory=%(ENV_ZEEK_LOG_PATH)s user=sensor diff --git a/sensor-iso/moloch/Dockerfile b/sensor-iso/moloch/Dockerfile index 8c0a3a836..8786de405 100644 --- a/sensor-iso/moloch/Dockerfile +++ b/sensor-iso/moloch/Dockerfile @@ -6,13 +6,27 @@ LABEL maintainer="malcolm.netsec@gmail.com" ENV DEBIAN_FRONTEND noninteractive -ENV MOLOCH_VERSION "2.3.2" +ENV MOLOCH_VERSION "2.4.0" ENV MOLOCHDIR "/opt/moloch" RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \ apt-get -q update && \ apt-get install -q -y --no-install-recommends \ - curl iproute2 python python-dev sudo ruby ruby-dev rubygems build-essential && \ + build-essential \ + curl \ + git-core \ + iproute2 \ + meson \ + ninja-build \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + ruby \ + ruby-dev \ + rubygems \ + sudo \ + wget && \ gem install --no-ri --no-rdoc fpm && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py index e02068be5..9c0e1092d 100755 --- a/shared/bin/configure-capture.py +++ b/shared/bin/configure-capture.py @@ -132,6 +132,7 @@ class Constants: MSG_IDENTIFY_NICS = 'Do you need help identifying network interfaces?' MSG_BACKGROUND_TITLE = 'Sensor Configuration' MSG_CONFIG_AUTOSTARTS = 'Specify autostart processes' + MSG_CONFIG_ZEEK_CARVED_SCANNERS = 'Specify scanners for Zeek-carved files' MSG_CONFIG_ZEEK_CARVING = 'Specify Zeek file carving mode' MSG_CONFIG_ZEEK_CARVING_MIMES = 'Specify file types to carve' MSG_CONFIG_CARVED_FILE_PRESERVATION = 'Specify which carved files to preserve' @@ -414,7 +415,7 @@ def main(): ##### sensor autostart services configuration ####################################################################################### while True: - # select processes for autostart + # select processes for autostart (except for the file scan ones, handle those with the file scanning stuff) autostart_choices = [] for k, v in sorted(capture_config_dict.items()): if k.startswith("AUTOSTART_"): @@ -509,6 +510,8 @@ def main(): zeek_carve_re = re.compile(r"(\bZEEK_EXTRACTOR_MODE)\s*=\s*.+?$") zeek_file_preservation_re = re.compile(r"(\bEXTRACTED_FILE_PRESERVATION)\s*=\s*.+?$") zeek_carve_override_re = re.compile(r"(\bZEEK_EXTRACTOR_OVERRIDE_FILE)\s*=\s*.*?$") + zeek_file_watch_re = re.compile(r"(\bZEEK_FILE_WATCH)\s*=\s*.+?$") + zeek_file_scanner_re = re.compile(r"(\bZEEK_FILE_SCAN_\w+)\s*=\s*.+?$") # get paths for captured PCAP and Zeek files while True: @@ -592,8 +595,26 @@ def main(): zeek_carve_mode = Constants.ZEEK_FILE_CARVING_MAPPED capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"] = Constants.ZEEK_FILE_CARVING_OVERRIDE_FILE + + # what to do with carved files if (zeek_carve_mode != Constants.ZEEK_FILE_CARVING_NONE): - # what to do with carved files + + # select engines for file scanning + scanner_choices = [] + for k, v in sorted(capture_config_dict.items()): + if k.startswith("ZEEK_FILE_SCAN_"): + scanner_choices.append((k, '', v.lower() == "true")) + code, scanner_tags = d.checklist(Constants.MSG_CONFIG_ZEEK_CARVED_SCANNERS, choices=scanner_choices) + if (code == Dialog.CANCEL or code == Dialog.ESC): + raise CancelledError + + for tag in [x[0] for x in scanner_choices]: + capture_config_dict[tag] = "false" + for tag in scanner_tags: + capture_config_dict[tag] = "true" + capture_config_dict["ZEEK_FILE_WATCH"] = "true" if (len(scanner_tags) > 0) else "false" + + # specify what to do with files that triggered the scanner engine(s) code, zeek_carved_file_preservation = d.radiolist(Constants.MSG_CONFIG_CARVED_FILE_PRESERVATION, choices=[(PRESERVE_QUARANTINED, 'Preserve only quarantined files', @@ -604,10 +625,14 @@ def main(): (PRESERVE_NONE, 'Preserve no files', (capture_config_dict["EXTRACTED_FILE_PRESERVATION"] == PRESERVE_NONE))]) - if (code == Dialog.CANCEL or code == Dialog.ESC): raise CancelledError + else: + # file carving disabled, so disable file scanning as well + for key in ["ZEEK_FILE_WATCH", "ZEEK_FILE_SCAN_CLAMAV", "ZEEK_FILE_SCAN_VTOT", "ZEEK_FILE_SCAN_MALASS", "ZEEK_FILE_SCAN_YARA"]: + capture_config_dict[key] = "false" + # reconstitute dictionary with user-specified values capture_config_dict["CAPTURE_INTERFACE"] = ",".join(selected_ifaces) capture_config_dict["CAPTURE_FILTER"] = capture_filter @@ -639,8 +664,14 @@ def main(): print(pcap_path_re.sub(r'\1="%s"' % capture_config_dict["PCAP_PATH"], line)) elif zeek_path_re.search(line) is not None: print(zeek_path_re.sub(r'\1="%s"' % capture_config_dict["ZEEK_LOG_PATH"], line)) + elif zeek_file_watch_re.search(line) is not None: + print(zeek_file_watch_re.sub(r"\1=%s" % capture_config_dict["ZEEK_FILE_WATCH"], line)) else: - print(line) + zeek_file_scanner_match = zeek_file_scanner_re.search(line) + if zeek_file_scanner_match is not None: + print(zeek_file_scanner_re.sub(r"\1=%s" % capture_config_dict[zeek_file_scanner_match.group(1)], line)) + else: + print(line) # write out file carving overrides if specified if (len(mime_tags) > 0) and (len(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]) > 0): diff --git a/shared/bin/sensor-init.sh b/shared/bin/sensor-init.sh index 9d2db0cca..6c48a8ba2 100755 --- a/shared/bin/sensor-init.sh +++ b/shared/bin/sensor-init.sh @@ -53,6 +53,11 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then chown -R 1000:1000 /opt/zeek/* [[ -f /opt/zeek/bin/zeek ]] && setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek fi + if [[ -d /opt/yara-rules ]]; then + mkdir -p /opt/yara-rules/custom + chown -R 1000:1000 /opt/yara-rules/custom + chmod -R 750 /opt/yara-rules/custom + fi # if the sensor needs to do clamav scanning, configure it to run as the sensor user if dpkg -s clamav >/dev/null 2>&1 ; then diff --git a/shared/bin/zeek_carve_logger.py b/shared/bin/zeek_carve_logger.py index 659314b95..b8bb23ddc 100755 --- a/shared/bin/zeek_carve_logger.py +++ b/shared/bin/zeek_carve_logger.py @@ -21,6 +21,7 @@ import time import zmq +from collections import defaultdict from contextlib import nullcontext from datetime import datetime from zeek_carve_utils import * @@ -56,6 +57,14 @@ def debug_toggle_handler(signum, frame): debug = not debug debugToggled = True +################################################################################################### +# +def same_file_or_dir(path1, path2): + try: + return os.path.samefile(path1, path2) + except: + return False + ################################################################################################### # main def main(): @@ -122,8 +131,8 @@ def main(): pathlib.Path(os.path.dirname(os.path.realpath(broSigLogSpec))).mkdir(parents=True, exist_ok=True) # create quarantine/preserved directories for preserved files (see preserveMode) - quarantineDir = os.path.join(args.baseDir, "quarantine") - preserveDir = os.path.join(args.baseDir, "preserved") + quarantineDir = os.path.join(args.baseDir, PRESERVE_QUARANTINED_DIR_NAME) + preserveDir = os.path.join(args.baseDir, PRESERVE_PRESERVED_DIR_NAME) if (args.preserveMode != PRESERVE_NONE) and (not os.path.isdir(quarantineDir)): if debug: eprint(f'Creating "{quarantineDir}" for quarantined files') pathlib.Path(quarantineDir).mkdir(parents=False, exist_ok=True) @@ -142,6 +151,9 @@ def main(): if debug: eprint(f"{scriptName}: bound sink port {SINK_PORT}") + scanners = set() + fileScanCounts = defaultdict(AtomicInt) + # open and write out header for our super legit zeek signature.log file with open(broSigLogSpec, 'w+', 1) if (broSigLogSpec is not None) else nullcontext() as broSigFile: if (broSigFile is not None): @@ -170,64 +182,94 @@ def main(): scanResult = None if verboseDebug: eprint(f"{scriptName}:\t🕑\t(recv)") - if isinstance(scanResult, dict) and all (k in scanResult for k in (FILE_SCAN_RESULT_FILE, - FILE_SCAN_RESULT_ENGINES, - FILE_SCAN_RESULT_HITS, - FILE_SCAN_RESULT_MESSAGE, - FILE_SCAN_RESULT_DESCRIPTION)): - - triggered = (scanResult[FILE_SCAN_RESULT_HITS] > 0) - fileName = scanResult[FILE_SCAN_RESULT_FILE] - - if triggered: - # this file had a "hit" in one of the virus engines, log it! - - # format the line as it should appear in the signatures log file - fileSpecFields = extracted_filespec_to_fields(fileName) - broLine = BroSignatureLine(ts=f"{fileSpecFields.time}", - uid=fileSpecFields.uid if fileSpecFields.uid is not None else '-', - note=ZEEK_SIGNATURE_NOTICE, - signature_id=scanResult[FILE_SCAN_RESULT_MESSAGE], - event_message=scanResult[FILE_SCAN_RESULT_DESCRIPTION], - sub_message=fileSpecFields.fid if fileSpecFields.fid is not None else os.path.basename(fileName), - signature_count=scanResult[FILE_SCAN_RESULT_HITS], - host_count=scanResult[FILE_SCAN_RESULT_ENGINES]) - broLineStr = str(broLine) - - # write broLineStr event line out to the signatures log file or to stdout - if (broSigFile is not None): - print(broLineStr, file=broSigFile, end='\n', flush=True) - else: - print(broLineStr, file=broSigFile, flush=True) + if isinstance(scanResult, dict): - # finally, what to do with the file itself - if os.path.isfile(fileName): - - if triggered and (args.preserveMode != PRESERVE_NONE): - # move triggering file to quarantine + # register/deregister scanners + if (FILE_SCAN_RESULT_SCANNER in scanResult): + scanner = scanResult[FILE_SCAN_RESULT_SCANNER].lower() + if scanner.startswith('-'): + if debug: eprint(f"{scriptName}:\t🙃\t{scanner[1:]}") try: - shutil.move(fileName, quarantineDir) - if debug: eprint(f"{scriptName}:\t⏩\t{fileName}") - except Exception as e: - eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}") - # hm move failed, delete it i guess? - os.remove(fileName) - - - elif (args.preserveMode == PRESERVE_ALL): - # move non-triggering file to preserved directory - try: - shutil.move(fileName, preserveDir) - if verboseDebug: eprint(f"{scriptName}:\t⏩\t{fileName}") - except Exception as e: - eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}") - # hm move failed, delete it i guess? - os.remove(fileName) - + scanners.remove(scanner[1:]) + except KeyError: + pass else: - # delete the file - os.remove(fileName) - if verboseDebug: eprint(f"{scriptName}:\t🚫\t{fileName}") + if debug and (scanner not in scanners): eprint(f"{scriptName}:\t🇷\t{scanner}") + scanners.add(scanner) + + # process scan results + if all (k in scanResult for k in (FILE_SCAN_RESULT_SCANNER, + FILE_SCAN_RESULT_FILE, + FILE_SCAN_RESULT_ENGINES, + FILE_SCAN_RESULT_HITS, + FILE_SCAN_RESULT_MESSAGE, + FILE_SCAN_RESULT_DESCRIPTION)): + + triggered = (scanResult[FILE_SCAN_RESULT_HITS] > 0) + fileName = scanResult[FILE_SCAN_RESULT_FILE] + fileNameBase = os.path.basename(fileName) + + # we may quarantine the file if fileScanCount < len(scanners), but we won't delete it so the rest of the scanners can find it + fileScanCount = fileScanCounts[fileNameBase].increment() + + if triggered: + # this file had a "hit" in one of the virus engines, log it! + + # format the line as it should appear in the signatures log file + fileSpecFields = extracted_filespec_to_fields(fileName) + broLine = BroSignatureLine(ts=f"{fileSpecFields.time}", + uid=fileSpecFields.uid if fileSpecFields.uid is not None else '-', + note=ZEEK_SIGNATURE_NOTICE, + signature_id=scanResult[FILE_SCAN_RESULT_MESSAGE], + event_message=scanResult[FILE_SCAN_RESULT_DESCRIPTION], + sub_message=fileSpecFields.fid if fileSpecFields.fid is not None else os.path.basename(fileName), + signature_count=scanResult[FILE_SCAN_RESULT_HITS], + host_count=scanResult[FILE_SCAN_RESULT_ENGINES]) + broLineStr = str(broLine) + + # write broLineStr event line out to the signatures log file or to stdout + if (broSigFile is not None): + print(broLineStr, file=broSigFile, end='\n', flush=True) + else: + print(broLineStr, file=broSigFile, flush=True) + + # finally, what to do with the file itself + if os.path.isfile(fileName): + + if triggered and (args.preserveMode != PRESERVE_NONE): + fileScanCounts.pop(fileNameBase, None) + + # move triggering file to quarantine + if not same_file_or_dir(fileName, os.path.join(quarantineDir, fileNameBase)): # unless it's already there + + try: + shutil.move(fileName, quarantineDir) + if debug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})") + except Exception as e: + eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}") + # hm move failed, delete it i guess? + os.remove(fileName) + + elif (fileScanCount >= len(scanners)): + fileScanCounts.pop(fileNameBase, None) + + if not same_file_or_dir(quarantineDir, os.path.dirname(fileName)): # don't move or delete if it's already quarantined + + if (args.preserveMode == PRESERVE_ALL): + # move non-triggering file to preserved directory + try: + shutil.move(fileName, preserveDir) + if verboseDebug: eprint(f"{scriptName}:\t⏩\t{fileName} ({fileScanCount}/{len(scanners)})") + except Exception as e: + eprint(f"{scriptName}:\t❗\t🚫\t{fileName} move exception: {e}") + # hm move failed, delete it i guess? + os.remove(fileName) + + else: + # delete the file + os.remove(fileName) + fileScanCounts.pop(fileNameBase, None) + if verboseDebug: eprint(f"{scriptName}:\t🚫\t{fileName} ({fileScanCount}/{len(scanners)})") # graceful shutdown if debug: diff --git a/shared/bin/zeek_carve_scanner.py b/shared/bin/zeek_carve_scanner.py index d612a81c5..903ed7d21 100755 --- a/shared/bin/zeek_carve_scanner.py +++ b/shared/bin/zeek_carve_scanner.py @@ -56,120 +56,157 @@ def debug_toggle_handler(signum, frame): debugToggled = True ################################################################################################### -def scanFileWorker(checkConnInfo): +# look for a file to scan (probably in its original directory, but possibly already moved to quarantine) +def locate_file(fileName): + global verboseDebug + + if fileName is not None: + + if os.path.isfile(fileName): + return fileName + + else: + for testPath in [PRESERVE_QUARANTINED_DIR_NAME, PRESERVE_PRESERVED_DIR_NAME]: + testFileName = os.path.join(os.path.join(os.path.dirname(os.path.realpath(fileName)), testPath), os.path.basename(fileName)) + if os.path.isfile(testFileName): + if verboseDebug: eprint(f"{scriptName}:\t⏩\t{testFileName}") + return testFileName + + return None + + +################################################################################################### +def scanFileWorker(checkConnInfo, carvedFileSub): global debug global verboseDebug global shuttingDown global scanWorkersCount scanWorkerId = scanWorkersCount.increment() # unique ID for this thread + scannerRegistered = False if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tstarted") - if isinstance(checkConnInfo, FileScanProvider): - - # initialize ZeroMQ context and socket(s) to receive filenames and send scan results - context = zmq.Context() - - # Socket to receive messages on - new_files_socket = context.socket(zmq.PULL) - new_files_socket.connect(f"tcp://localhost:{VENTILATOR_PORT}") - new_files_socket.RCVTIMEO = 5000 - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tbound to ventilator at {VENTILATOR_PORT}") - - # Socket to send messages to - scanned_files_socket = context.socket(zmq.PUSH) - scanned_files_socket.connect(f"tcp://localhost:{SINK_PORT}") - # todo: do I want to set this? probably not, since what else would we do if we can't send? just block - # scanned_files_socket.SNDTIMEO = 5000 - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tconnected to sink at {SINK_PORT}") + try: + if isinstance(checkConnInfo, FileScanProvider): - fileName = None - retrySubmitFile = False # todo: maximum file retry count? + # initialize ZeroMQ context and socket(s) to send scan results + context = zmq.Context() - # loop forever, or until we're told to shut down - while not shuttingDown: + # Socket to send messages to + scanned_files_socket = context.socket(zmq.PUSH) + scanned_files_socket.connect(f"tcp://localhost:{SINK_PORT}") + # todo: do I want to set this? probably not, since what else would we do if we can't send? just block + # scanned_files_socket.SNDTIMEO = 5000 + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tconnected to sink at {SINK_PORT}") - if retrySubmitFile and (fileName is not None) and os.path.isfile(fileName): - # we were unable to submit the file for processing, so try again - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔃\t{fileName}") + fileName = None + retrySubmitFile = False # todo: maximum file retry count? - else: - retrySubmitFile = False + # loop forever, or until we're told to shut down + while not shuttingDown: - # accept a filename from new_files_socket - try: - fileName = new_files_socket.recv_string() - except zmq.Again as timeout: - # no file received due to timeout, we'll go around and try again - if verboseDebug: eprint(f"{scriptName}[{scanWorkerId}]:\t🕑\t(recv)") - fileName = None + # "register" this scanner with the logger + while (not scannerRegistered) and (not shuttingDown): + try: + scanned_files_socket.send_string(json.dumps({FILE_SCAN_RESULT_SCANNER : checkConnInfo.scanner_name()})) + scannerRegistered = True + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🇷\t{checkConnInfo.scanner_name()}") - if (fileName is not None) and os.path.isfile(fileName): + except zmq.Again as timeout: + # todo: what to do here? + if verboseDebug: eprint(f"{scriptName}[{scanWorkerId}]:\t🕑\t{checkConnInfo.scanner_name()} 🇷") - # file exists, submit for scanning - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileName}") - requestComplete = False - scanResult = None - scan = AnalyzerScan(provider=checkConnInfo, name=fileName, - submissionResponse=checkConnInfo.submit(fileName=fileName, block=False)) + if shuttingDown: + break - if scan.submissionResponse is not None: - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔍\t{fileName}") + if retrySubmitFile and (fileName is not None) and (locate_file(fileName) is not None): + # we were unable to submit the file for processing, so try again + time.sleep(1) + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔃\t{fileName}") - # file was successfully submitted and is now being scanned + else: retrySubmitFile = False - requestComplete = False + # read a filename from the subscription + fileName = carvedFileSub.Pull(scanWorkerId=scanWorkerId) - # todo: maximum time we wait for a single file to be scanned? - while (not requestComplete) and (not shuttingDown): + fileName = locate_file(fileName) + if (fileName is not None) and os.path.isfile(fileName): - # wait a moment then check to see if the scan is complete - time.sleep(scan.provider.check_interval()) - response = scan.provider.check_result(scan.submissionResponse) + # file exists, submit for scanning + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileName}") + requestComplete = False + scanResult = None + scan = AnalyzerScan(provider=checkConnInfo, name=fileName, + submissionResponse=checkConnInfo.submit(fileName=fileName, block=False)) - if isinstance(response, AnalyzerResult): + if scan.submissionResponse is not None: + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🔍\t{fileName}") - # whether the scan has completed - requestComplete = response.finished + # file was successfully submitted and is now being scanned + retrySubmitFile = False + requestComplete = False - if response.success: - # successful scan, report the scan results - scanResult = response.result + # todo: maximum time we wait for a single file to be scanned? + while (not requestComplete) and (not shuttingDown): - elif isinstance(response.result, dict) and ("error" in response.result): - # scan errored out, report the error - scanResult = response.result["error"] - eprint(f"{scriptName}[{scanWorkerId}]:\t❗\t{fileName} {scanResult}") + # wait a moment then check to see if the scan is complete + time.sleep(scan.provider.check_interval()) + response = scan.provider.check_result(scan.submissionResponse) - else: - # result is unrecognizable - scanResult = "Invalid scan result format" - eprint(f"{scriptName}[{scanWorkerId}]:\t❗\t{fileName} {scanResult}") + if isinstance(response, AnalyzerResult): - else: - # impossibru! abandon ship for this file? - # todo? what else? touch it? - requestComplete = True - scanResult = "Error checking results" - eprint(f"{scriptName}[{scanWorkerId}]:\t❗{fileName} {scanResult}") + # whether the scan has completed + requestComplete = response.finished - else: - # we were denied (rate limiting, probably), so we'll need wait for a slot to clear up - retrySubmitFile = True + if response.success: + # successful scan, report the scan results + scanResult = response.result - if requestComplete and (scanResult is not None): - try: - # Send results to sink - scanned_files_socket.send_string(json.dumps(scan.provider.format(fileName, scanResult))) - if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t✅\t{fileName}") + elif isinstance(response.result, dict) and ("error" in response.result): + # scan errored out, report the error + scanResult = response.result["error"] + eprint(f"{scriptName}[{scanWorkerId}]:\t❗\t{fileName} {scanResult}") - except zmq.Again as timeout: - # todo: what to do here? - if verboseDebug: eprint(f"{scriptName}[{scanWorkerId}]:\t🕑\t{fileName}") + else: + # result is unrecognizable + scanResult = "Invalid scan result format" + eprint(f"{scriptName}[{scanWorkerId}]:\t❗\t{fileName} {scanResult}") - else: - eprint(f"{scriptName}[{scanWorkerId}]:\tinvalid scanner provider specified") + else: + # impossibru! abandon ship for this file? + # todo? what else? touch it? + requestComplete = True + scanResult = "Error checking results" + eprint(f"{scriptName}[{scanWorkerId}]:\t❗{fileName} {scanResult}") + + else: + # we were denied (rate limiting, probably), so we'll need wait for a slot to clear up + retrySubmitFile = True + + if requestComplete and (scanResult is not None): + try: + # Send results to sink + scanned_files_socket.send_string(json.dumps(scan.provider.format(fileName, scanResult))) + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t✅\t{fileName}") + + except zmq.Again as timeout: + # todo: what to do here? + if verboseDebug: eprint(f"{scriptName}[{scanWorkerId}]:\t🕑\t{fileName}") + + else: + eprint(f"{scriptName}[{scanWorkerId}]:\tinvalid scanner provider specified") + + finally: + # "unregister" this scanner with the logger + if scannerRegistered: + try: + scanned_files_socket.send_string(json.dumps({FILE_SCAN_RESULT_SCANNER : f"-{checkConnInfo.scanner_name()}"})) + scannerRegistered = False + if debug: eprint(f"{scriptName}[{scanWorkerId}]:\t🙃\t{checkConnInfo.scanner_name()}") + except zmq.Again as timeout: + # todo: what to do here? + if verboseDebug: eprint(f"{scriptName}[{scanWorkerId}]:\t🕑\t{checkConnInfo.scanner_name()} 🙃") if debug: eprint(f"{scriptName}[{scanWorkerId}]:\tfinished") @@ -192,8 +229,10 @@ def main(): parser.add_argument('--malass-limit', dest='malassLimit', help="Malass maximum concurrent scans", metavar='', type=int, default=MAL_MAX_REQS, required=False) parser.add_argument('--vtot-api', dest='vtotApi', help="VirusTotal API key", metavar='', type=str, required=False) parser.add_argument('--vtot-req-limit', dest='vtotReqLimit', help="VirusTotal requests per minute limit", metavar='', type=int, default=VTOT_MAX_REQS, required=False) - parser.add_argument('--clamav', dest='enableClamAv', metavar='true|false', help="Enable ClamAV (if VirusTotal and Malass are unavailable)", type=str2bool, nargs='?', const=True, default=False, required=False) + parser.add_argument('--clamav', dest='enableClamAv', metavar='true|false', help="Enable ClamAV", type=str2bool, nargs='?', const=True, default=False, required=False) parser.add_argument('--clamav-socket', dest='clamAvSocket', help="ClamAV socket filename", metavar='', type=str, required=False, default=None) + parser.add_argument('--yara', dest='enableYara', metavar='true|false', help="Enable Yara", type=str2bool, nargs='?', const=True, default=False, required=False) + parser.add_argument('--yara-custom-only', dest='yaraCustomOnly', metavar='true|false', help="Ignore default Yara rules", type=str2bool, nargs='?', const=True, default=False, required=False) try: parser.error = parser.exit @@ -228,13 +267,23 @@ def main(): checkConnInfo = MalassScan(args.malassHost, args.malassPort, reqLimit=args.malassLimit) elif (isinstance(args.vtotApi, str) and (len(args.vtotApi) > 1) and (args.vtotReqLimit > 0)): checkConnInfo = VirusTotalSearch(args.vtotApi, reqLimit=args.vtotReqLimit) + elif args.enableYara: + yaraDirs = [] + if (not args.yaraCustomOnly): + yaraDirs.append(YARA_RULES_DIR) + yaraDirs.append(YARA_CUSTOM_RULES_DIR) + checkConnInfo = YaraScan(debug=debug, verboseDebug=verboseDebug, rulesDirs=yaraDirs) else: if not args.enableClamAv: eprint('No scanner specified, defaulting to ClamAV') checkConnInfo = ClamAVScan(debug=debug, verboseDebug=verboseDebug, socketFileName=args.clamAvSocket) + carvedFileSub = CarvedFileSubscriberThreaded(debug=debug, verboseDebug=verboseDebug, + host='localhost', port=VENTILATOR_PORT, + scriptName=scriptName) + # start scanner threads which will pull filenames to be scanned and send the results to the logger - scannerThreads = ThreadPool(checkConnInfo.max_requests(), scanFileWorker, ([checkConnInfo])) + scannerThreads = ThreadPool(checkConnInfo.max_requests(), scanFileWorker, ([checkConnInfo, carvedFileSub])) while (not shuttingDown): if pdbFlagged: pdbFlagged = False diff --git a/shared/bin/zeek_carve_utils.py b/shared/bin/zeek_carve_utils.py index a959e8ec0..cdc8d5d6f 100644 --- a/shared/bin/zeek_carve_utils.py +++ b/shared/bin/zeek_carve_utils.py @@ -5,12 +5,15 @@ import clamd import hashlib +import json import malass_client import os import re import requests import sys import time +import yara +import zmq from abc import ABC, abstractmethod from bs4 import BeautifulSoup @@ -25,6 +28,7 @@ ################################################################################################### VENTILATOR_PORT = 5987 SINK_PORT = 5988 +TOPIC_FILE_SCAN = "file" ################################################################################################### # modes for file preservation settings @@ -32,7 +36,11 @@ PRESERVE_ALL = "all" PRESERVE_NONE = "none" +PRESERVE_QUARANTINED_DIR_NAME = "quarantine" +PRESERVE_PRESERVED_DIR_NAME = "preserved" + ################################################################################################### +FILE_SCAN_RESULT_SCANNER = "scanner" FILE_SCAN_RESULT_FILE = "file" FILE_SCAN_RESULT_ENGINES = "engines" FILE_SCAN_RESULT_HITS = "hits" @@ -71,6 +79,15 @@ CLAM_ENGINE_ID = 'ClamAV' CLAM_FOUND_KEY = 'FOUND' +################################################################################################### +# Yara Interface +YARA_RULES_DIR = os.path.join(os.getenv('YARA_RULES_DIR', "/yara-rules"), '') +YARA_CUSTOM_RULES_DIR = os.path.join(YARA_RULES_DIR, "custom") +YARA_SUBMIT_TIMEOUT_SEC = 60 +YARA_ENGINE_ID = 'Yara' +YARA_MAX_REQS = 8 # maximum scanning threads concurrently +YARA_CHECK_INTERVAL = 0.1 + ################################################################################################### @@ -217,9 +234,53 @@ def value(self): with self.lock: return self.val.value +################################################################################################### +class CarvedFileSubscriberThreaded: + + # --------------------------------------------------------------------------------- + # constructor + def __init__(self, debug=False, verboseDebug=False, host="localhost", port=VENTILATOR_PORT, context=None, topic='', rcvTimeout=5000, scriptName=''): + self.debug = debug + self.verboseDebug = verboseDebug + self.scriptName = scriptName + + self.lock = Lock() + + # initialize ZeroMQ context and socket(s) to receive filenames and send scan results + self.context = context if (context is not None) else zmq.Context() + + # Socket to receive messages on + self.newFilesSocket = self.context.socket(zmq.SUB) + self.newFilesSocket.connect(f"tcp://{host}:{port}") + self.newFilesSocket.setsockopt(zmq.SUBSCRIBE, bytes(topic, encoding='ascii')) + self.newFilesSocket.RCVTIMEO = rcvTimeout + if self.debug: eprint(f"{self.scriptName}:\tbound to ventilator at {port}") + + # --------------------------------------------------------------------------------- + def Pull(self, scanWorkerId=0): + + with self.lock: + # accept a filename from newFilesSocket + try: + filename = self.newFilesSocket.recv_string() + except zmq.Again as timeout: + # no file received due to timeout, return "None" which means no file available + filename = None + + if self.verboseDebug: + eprint(f"{self.scriptName}[{scanWorkerId}]:\t{'📨' if (filename is not None) else '🕑'}\t{filename if (filename is not None) else '(recv)'}") + + return filename + ################################################################################################### class FileScanProvider(ABC): + @staticmethod + @abstractmethod + def scanner_name(cls): + # returns this scanner name + pass + @staticmethod @abstractmethod def max_requests(cls): @@ -261,6 +322,10 @@ def __init__(self, apiKey, reqLimit=VTOT_MAX_REQS, reqLimitSec=VTOT_MAX_SEC): self.reqLimit = reqLimit self.reqLimitSec = reqLimitSec + @staticmethod + def scanner_name(): + return 'virustotal' + @staticmethod def max_requests(): return VTOT_MAX_REQS @@ -274,7 +339,7 @@ def check_interval(): # VirusTotalSearch does the request and gets the response immediately; # the subsequent call to check_result (using submit's response as input) # will always return "True" since the work has already been done - def submit(self, fileName=None, block=False, timeout=None): + def submit(self, fileName=None, block=False, timeout=0): if timeout is None: timeout = self.reqLimitSec+5 @@ -340,7 +405,8 @@ def check_result(self, submissionResponse): # static method for formatting the response JSON (from requests.get) as a dict @staticmethod def format(fileName, response): - result = {FILE_SCAN_RESULT_FILE : fileName, + result = {FILE_SCAN_RESULT_SCANNER : VirusTotalSearch.scanner_name(), + FILE_SCAN_RESULT_FILE : fileName, FILE_SCAN_RESULT_ENGINES : 0, FILE_SCAN_RESULT_HITS : 0, FILE_SCAN_RESULT_MESSAGE : None, @@ -399,6 +465,10 @@ def __init__(self, host, port, reqLimit=MAL_MAX_REQS): self.transactionIdToFilenameDict = defaultdict(str) self.scanningFilesCount = AtomicInt(value=0) + @staticmethod + def scanner_name(): + return 'malass' + @staticmethod def max_requests(): return MAL_MAX_REQS @@ -522,7 +592,8 @@ def check_result(self, transactionId): # static method for formatting the response summaryDict (from check_result) @staticmethod def format(fileName, response): - result = {FILE_SCAN_RESULT_FILE : fileName, + result = {FILE_SCAN_RESULT_SCANNER : MalassScan.scanner_name(), + FILE_SCAN_RESULT_FILE : fileName, FILE_SCAN_RESULT_ENGINES : 0, FILE_SCAN_RESULT_HITS : 0, FILE_SCAN_RESULT_MESSAGE : None, @@ -567,6 +638,10 @@ def __init__(self, debug=False, verboseDebug=False, socketFileName=None): self.verboseDebug = verboseDebug self.socketFileName = socketFileName + @staticmethod + def scanner_name(): + return 'clamav' + @staticmethod def max_requests(): return CLAM_MAX_REQS @@ -576,7 +651,7 @@ def check_interval(): return CLAM_CHECK_INTERVAL # --------------------------------------------------------------------------------- - # upload a file to scan with ClamAV, respecting rate limiting. return submitted transaction ID + # submit a file to scan with ClamAV, respecting rate limiting. return scan result def submit(self, fileName=None, block=False, timeout=CLAM_SUBMIT_TIMEOUT_SEC): clamavResult = AnalyzerResult() allowed = False @@ -639,7 +714,8 @@ def check_result(self, clamavResult): # static method for formatting the response summaryDict (from check_result) @staticmethod def format(fileName, response): - result = {FILE_SCAN_RESULT_FILE : fileName, + result = {FILE_SCAN_RESULT_SCANNER : ClamAVScan.scanner_name(), + FILE_SCAN_RESULT_FILE : fileName, FILE_SCAN_RESULT_ENGINES : 1, FILE_SCAN_RESULT_HITS : 0, FILE_SCAN_RESULT_MESSAGE : None, @@ -670,4 +746,126 @@ def format(fileName, response): else: result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}" + return result + +################################################################################################### +# class for scanning a file with Yara +class YaraScan(FileScanProvider): + + # --------------------------------------------------------------------------------- + # constructor + def __init__(self, debug=False, verboseDebug=False, rulesDirs=[]): + self.scanningFilesCount = AtomicInt(value=0) + self.debug = debug + self.verboseDebug = verboseDebug + self.ruleFilespecs = {} + for yaraDir in rulesDirs: + for root, dirs, files in os.walk(yaraDir): + for file in files: + # skip hidden, backup or system related files + if file.startswith(".") or file.startswith("~") or file.startswith("_"): + continue + filename = os.path.join(root, file) + extension = os.path.splitext(file)[1].lower() + try: + testCompile = yara.compile(filename) + self.ruleFilespecs[filename] = filename + except yara.SyntaxError as e: + if self.debug: eprint(f'{get_ident()} Ignored Yara compile error in {filename}: {e}') + if self.verboseDebug: + eprint(f"{get_ident()}: Initializing Yara with {len(self.ruleFilespecs)} rules files: {self.ruleFilespecs}") + elif self.debug: + eprint(f"{get_ident()}: Initializing Yara with {len(self.ruleFilespecs)} rules files") + self.compiledRules = yara.compile(filepaths = self.ruleFilespecs) + + @staticmethod + def scanner_name(): + return 'yara' + + @staticmethod + def max_requests(): + return YARA_MAX_REQS + + @staticmethod + def check_interval(): + return YARA_CHECK_INTERVAL + + # --------------------------------------------------------------------------------- + # submit a file to scan with Yara, respecting rate limiting. return scan result + def submit(self, fileName=None, block=False, timeout=YARA_SUBMIT_TIMEOUT_SEC): + yaraResult = AnalyzerResult() + allowed = False + matches = [] + + # timeout only applies if block=True + timeoutTime = int(time.time()) + timeout + + # while limit only repeats if block=True + while (not allowed) and (not yaraResult.finished): + + # first make sure we haven't exceeded rate limits + if (self.scanningFilesCount.increment() <= YARA_MAX_REQS): + # we've got fewer than the allowed requests open, so we're good to go! + allowed = True + else: + self.scanningFilesCount.decrement() + + if allowed: + try: + if self.verboseDebug: eprint(f'{get_ident()} Yara scanning: {fileName}') + yaraResult.result = self.compiledRules.match(fileName) + if self.verboseDebug: eprint(f'{get_ident()} Yara scan result: {yaraResult.result}') + yaraResult.success = (yaraResult.result is not None) + yaraResult.finished = True + except Exception as e: + if yaraResult.result is None: + yaraResult.result = str(e) + if self.debug: eprint(f'{get_ident()} Yara scan error: {yaraResult.result}') + finally: + self.scanningFilesCount.decrement() + + elif block and (nowTime < timeoutTime): + # rate limited, wait for a bit and come around and try again + time.sleep(1) + + else: + break + + return yaraResult + + # --------------------------------------------------------------------------------- + # return the result of the previously scanned file + def check_result(self, yaraResult): + return yaraResult if isinstance(yaraResult, AnalyzerResult) else AnalyzerResult(finished=True, success=False, result=None) + + # --------------------------------------------------------------------------------- + # static method for formatting the response summaryDict (from check_result) + @staticmethod + def format(fileName, response): + result = {FILE_SCAN_RESULT_SCANNER : YaraScan.scanner_name(), + FILE_SCAN_RESULT_FILE : fileName, + FILE_SCAN_RESULT_ENGINES : 1, + FILE_SCAN_RESULT_HITS : 0, + FILE_SCAN_RESULT_MESSAGE : None, + FILE_SCAN_RESULT_DESCRIPTION : None} + + if isinstance(response, AnalyzerResult): + resp = response.result + else: + resp = response + + if isinstance(resp, list): + hits = [match.rule for match in resp if isinstance(match, yara.Match)] + result[FILE_SCAN_RESULT_HITS] = len(hits) + if (len(hits) > 0): + cnt = Counter(hits) + # short message is most common signature name (todo: they won't have duplicate names, so I guess this is just going to take the first...) + result[FILE_SCAN_RESULT_MESSAGE] = cnt.most_common(1)[0][0] + # long description is list of the signature names and the engines which generated them + result[FILE_SCAN_RESULT_DESCRIPTION] = ";".join([f"{x}<{YARA_ENGINE_ID}>" for x in hits]) + + else: + result[FILE_SCAN_RESULT_MESSAGE] = "Error or invalid response" + result[FILE_SCAN_RESULT_DESCRIPTION] = f"{resp}" + return result \ No newline at end of file diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index 64c729d76..b8ce2e6cd 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -53,7 +53,7 @@ def __init__(self): # Socket to send messages on if debug: eprint(f"{scriptName}:\tbinding ventilator port {VENTILATOR_PORT}") - self.ventilator_socket = self.context.socket(zmq.PUSH) + self.ventilator_socket = self.context.socket(zmq.PUB) self.ventilator_socket.bind(f"tcp://*:{VENTILATOR_PORT}") # todo: do I want to set this? probably not since this guy's whole job is to send diff --git a/yara/rules/.gitignore b/yara/rules/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/yara/rules/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index 1b43983e0..0b4810086 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -82,7 +82,7 @@ redef ignore_checksums = T; @if (!disable_wireguard) @load ./spicy-noise - event zeek_init() { + event zeek_init() &priority=-5 { if (disable_wireguard_transport_packets) { Log::remove_default_filter(WireGuard::WGLOG); Log::add_filter(WireGuard::WGLOG,