Malcolm v3.4.0 development (#176)

Malcolm v3.4.0 development (#176)

- Component version updates
    * Arkime [v3.3.1](https://github.com/arkime/arkime/blob/v3.1.1/CHANGELOG#L25-L143) (from v2.7.1)
    * [Zeek](https://github.com/zeek/zeek/releases) v4.1.1 (from v4.0.4)
        - [Zeek v4.1 Feature Release](https://zeek.org/2021/08/13/zeek-4-1-feature-release/)
    * [Spicy](https://github.com/zeek/spicy/releases) v1.3.0 (from v1.2.1)
    * [Yara](https://github.com/VirusTotal/yara/releases) v4.1.3 (from v4.1.2)
    * [Capa](https://github.com/mandiant/capa/releases) v3.0.3 (from v3.0.2)
    * [Debian](https://www.debian.org/releases/bullseye/amd64/release-notes/) v10 to v11 (for ISO images)
- Added [GitHub actions](https://github.com/cisagov/Malcolm/) for building the Malcolm Docker images on GitHub and pushing them to GHCR
- Moved common Logstash [Ruby code](https://www.elastic.co/guide/en/logstash/current/plugins-filters-ruby.html) to [file-based scripting](https://www.elastic.co/blog/moving-ruby-code-out-of-logstash-pipeline)
- Use standard stunnel package in NGINX proxy container rather than building from source
- Switched from CLANG to GCC build toolchain for Zeek and Spicy plugins
- Replaced [LXDE](http://www.lxde.org/) desktop environment with [XFCE](https://www.xfce.org/) (for ISO images)
- Renamed various fields to align with Arkime's gradual adoption of the Elastic Common Schema
- Added [parser support](https://github.com/zeek/spicy-analyzers/tree/main/analyzer/stun) and dashboard for the [STUN (Session Traversal Utilities for NAT) protocol](https://datatracker.ietf.org/doc/html/rfc3489)
- Further improved capabilities for tagging ICS traffic
    * Logs from known ICS protocols how have `ics` added to the `tags` field
    * Logs identified by "ICS best guess" lookups now have `ics_best_guess` added to the `tags` field
    * "ICS best guess" lookups have been augmented with a MAC address lookup table of ICS hardware vendors
    * ICS-related overview dashboards have been updated accordingly

.dockerignore

@@ -16,8 +16,8 @@ elasticsearch
 elasticsearch-backup
 malcolm-iso
 sensor-iso
-moloch-logs
-moloch-raw
+arkime-logs
+arkime-raw
 nginx/nginx_ldap.conf
 pcap
 scripts

.github/workflows/arkime-build-and-push-ghcr.yml

@@ -0,0 +1,61 @@
+name: arkime-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'arkime/**'
+      - 'Dockerfiles/arkime.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/arkime.Dockerfile
+          build-args: |
+            MAXMIND_GEOIP_DB_LICENSE_KEY=${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/arkime:${{ steps.extract_branch.outputs.branch }}

.github/workflows/elasticsearch-build-and-push-ghcr.yml

@@ -0,0 +1,58 @@
+name: elasticsearch-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'Dockerfiles/elasticsearch.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/elasticsearch.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/elasticsearch-od:${{ steps.extract_branch.outputs.branch }}

.github/workflows/file-monitor-build-and-push-ghcr.yml

@@ -0,0 +1,59 @@
+name: file-monitor-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'file-monitor/**'
+      - 'Dockerfiles/file-monitor.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/file-monitor.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/file-monitor:${{ steps.extract_branch.outputs.branch }}

.github/workflows/file-upload-build-and-push-ghcr.yml

@@ -0,0 +1,59 @@
+name: file-upload-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'file-upload/**'
+      - 'Dockerfiles/file-upload.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/file-upload.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/file-upload:${{ steps.extract_branch.outputs.branch }}

.github/workflows/filebeat-build-and-push-ghcr.yml

@@ -0,0 +1,59 @@
+name: filebeat-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'filebeat/**'
+      - 'Dockerfiles/filebeat.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/filebeat.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/filebeat-oss:${{ steps.extract_branch.outputs.branch }}

.github/workflows/freq-build-and-push-ghcr.yml

@@ -0,0 +1,59 @@
+name: freq-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+    paths:
+      - 'freq-server/**'
+      - 'Dockerfiles/freq.Dockerfile'
+      - 'shared/bin/*'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/freq.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/freq:${{ steps.extract_branch.outputs.branch }}