Skip to content

Commit

Permalink
Merge pull request #2 from waycarbon/feature/fangfrisch-extra-defs
Browse files Browse the repository at this point in the history 
Feature/fangfrisch extra defs
  • Loading branch information
@gchamon
gchamon authored Mar 3, 2023
2 parents 94609bf + 750f7f0 commit 4cf9df2
Show file tree
Hide file tree
Showing 9 changed files with 536 additions and 178 deletions.
2 changes: 1 addition & 1 deletion .envrc.local.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,5 @@ export TEST_KEY=""
# export AV_STATUS_SNS_PUBLISH_INFECTED
# export AV_TIMESTAMP_METADATA
# export CLAMAVLIB_PATH
# export CLAMSCAN_PATH
# export CLAMDSCAN_PATH
# export FRESHCLAM_PATH
169 changes: 101 additions & 68 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,86 +1,119 @@
FROM public.ecr.aws/lambda/python:3.7 AS cli_deps

COPY requirements-cli.txt requirements-cli.txt
RUN mkdir -p /opt/app/cli \
&& pip3 install --requirement requirements-cli.txt --target /opt/app/cli \
&& rm -rf /root/.cache/pip

FROM amazonlinux:2

# Set up working directories
RUN mkdir -p /opt/app
RUN mkdir -p /opt/app/build
RUN mkdir -p /opt/app/bin/

# Copy in the lambda source
WORKDIR /opt/app
COPY ./*.py /opt/app/
COPY requirements.txt /opt/app/requirements.txt
RUN mkdir -p \
/opt/app \
/opt/app/build \
/opt/app/bin \
/opt/app/python_deps \
/opt/app/cli

# Install packages
RUN yum update -y
RUN amazon-linux-extras install epel -y
RUN yum install -y cpio yum-utils tar.x86_64 gzip zip python3-pip

# This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
RUN pip3 install -r requirements.txt
RUN rm -rf /root/.cache/pip
RUN yum update -y \
&& amazon-linux-extras install epel -y \
&& yum install -y \
cpio \
yum-utils \
tar.x86_64 \
gzip \
zip \
python3-pip \
shadow-utils.x86_64 \
&& yum clean all \
&& rm -rf /var/cache/yum

# Download libraries we need to run in lambda
WORKDIR /tmp
RUN yumdownloader -x \*i686 --archlist=x86_64 clamav
RUN rpm2cpio clamav-0*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 clamav-lib
RUN rpm2cpio clamav-lib*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 clamav-update
RUN rpm2cpio clamav-update*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 json-c
RUN rpm2cpio json-c*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 pcre2
RUN rpm2cpio pcre*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 libtool-ltdl
RUN rpm2cpio libtool-ltdl*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 libxml2
RUN rpm2cpio libxml2*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 bzip2-libs
RUN rpm2cpio bzip2-libs*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 xz-libs
RUN rpm2cpio xz-libs*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 libprelude
RUN rpm2cpio libprelude*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 gnutls
RUN rpm2cpio gnutls*.rpm | cpio -vimd

RUN yumdownloader -x \*i686 --archlist=x86_64 nettle
RUN rpm2cpio nettle*.rpm | cpio -vimd


# Copy over the binaries and libraries
RUN cp /tmp/usr/bin/clamscan /tmp/usr/bin/freshclam /tmp/usr/lib64/* /usr/lib64/libpcre.so.1 /opt/app/bin/
RUN yumdownloader -x \*i686 --archlist=x86_64 \
clamav \
clamav-lib \
clamav-update \
clamav-scanner-systemd \
elfutils-libs \
json-c \
lz4 \
pcre2 \
systemd-libs \
libtool-ltdl \
libxml2 \
bzip2-libs \
xz-libs \
libprelude \
gnutls \
nettle \
&& rpm2cpio clamav-0*.rpm | cpio -vimd \
&& rpm2cpio clamav-lib*.rpm | cpio -vimd \
&& rpm2cpio clamav-update*.rpm | cpio -vimd \
&& rpm2cpio json-c*.rpm | cpio -vimd \
&& rpm2cpio pcre*.rpm | cpio -vimd \
&& rpm2cpio libtool-ltdl*.rpm | cpio -vimd \
&& rpm2cpio libxml2*.rpm | cpio -vimd \
&& rpm2cpio bzip2-libs*.rpm | cpio -vimd \
&& rpm2cpio xz-libs*.rpm | cpio -vimd \
&& rpm2cpio libprelude*.rpm | cpio -vimd \
&& rpm2cpio gnutls*.rpm | cpio -vimd \
&& rpm2cpio nettle*.rpm | cpio -vimd \
&& rpm2cpio clamd-0*.rpm | cpio -idmv \
&& rpm2cpio elfutils-libs*.rpm | cpio -idmv \
&& rpm2cpio lz4*.rpm | cpio -idmv \
&& rpm2cpio systemd-libs*.rpm | cpio -idmv \
&& cp -r \
/tmp/usr/bin/clamdscan \
/tmp/usr/sbin/clamd \
/tmp/usr/bin/freshclam \
/tmp/usr/lib64/* \
/usr/lib64/libpcre.so* \
/opt/app/bin/ \
&& rm -rf /tmp/usr

# Fix the freshclam.conf settings
RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf
RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
RUN echo "ScriptedUpdates no" >> /opt/app/bin/freshclam.conf
RUN echo "DatabaseDirectory /var/lib/clamav" >> /opt/app/bin/freshclam.conf

RUN yum install shadow-utils.x86_64 -y

RUN groupadd clamav
RUN useradd -g clamav -s /bin/false -c "Clam Antivirus" clamav
RUN useradd -g clamav -s /bin/false -c "Clam Antivirus" clamupdate
RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf \
&& echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf \
&& echo "ScriptedUpdates no" >> /opt/app/bin/freshclam.conf \
&& echo "DatabaseDirectory /var/lib/clamav" >> /opt/app/bin/freshclam.conf
# clamd conf with hardened configs to avoid false positives
RUN echo "DatabaseDirectory /tmp/clamav_defs" > /opt/app/bin/scan.conf \
&& echo "PidFile /tmp/clamd.pid" >> /opt/app/bin/scan.conf \
&& echo "LogFile /tmp/clamd.log" >> /opt/app/bin/scan.conf \
&& echo "LocalSocket /tmp/clamd.sock" >> /opt/app/bin/scan.conf \
&& echo "FixStaleSocket yes" >> /opt/app/bin/scan.conf \
&& echo "DetectPUA yes" >> /opt/app/bin/scan.conf \
&& echo "ExcludePUA PUA.Win.Packer" >> /opt/app/bin/scan.conf \
&& echo "ExcludePUA PUA.Win.Trojan.Packed" >> /opt/app/bin/scan.conf \
&& echo "ExcludePUA PUA.Win.Trojan.Molebox" >> /opt/app/bin/scan.conf \
&& echo "ExcludePUA PUA.Win.Packer.Upx" >> /opt/app/bin/scan.conf \
&& echo "ExcludePUA PUA.Doc.Packed" >> /opt/app/bin/scan.conf

RUN groupadd clamav \
&& useradd -g clamav -s /bin/false -c "Clam Antivirus" clamav \
&& useradd -g clamav -s /bin/false -c "Clam Antivirus" clamupdate

ENV LD_LIBRARY_PATH=/opt/app/bin
RUN ldconfig

# Create the zip file
# Copy in the lambda source
WORKDIR /opt/app
RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
COPY requirements.txt /opt/app/requirements.txt

# This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
RUN pip3 install --requirement requirements.txt --target /opt/app/python_deps \
&& rm -rf /root/.cache/pip

# Copy fangfrisch CLI from lambda image
COPY --from=cli_deps /opt/app/cli /opt/app/cli

WORKDIR /usr/local/lib/python3.7/site-packages
RUN zip -r9 /opt/app/build/lambda.zip *
# Create the zip file
COPY ./*.py /opt/app/
COPY fangfrisch.conf /opt/app/fangfrisch.conf
RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py *.conf bin cli \
&& cd /opt/app/python_deps \
&& zip -r9 /opt/app/build/lambda.zip *

WORKDIR /opt/app
51 changes: 26 additions & 25 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,31 +81,32 @@ can cause a continuous loop of scanning if improperly configured.
Runtime configuration is accomplished using environment variables. See
the table below for reference.

| Variable | Description | Default | Required |
| --- | --- | --- | --- |
| AV_DEFINITION_S3_BUCKET | Bucket containing antivirus definition files | | Yes |
| AV_DEFINITION_S3_PREFIX | Prefix for antivirus definition files | clamav_defs | No |
| AV_DEFINITION_PATH | Path containing files at runtime | /tmp/clamav_defs | No |
| AV_SCAN_START_SNS_ARN | SNS topic ARN to publish notification about start of scan | | No |
| AV_SCAN_START_METADATA | The tag/metadata indicating the start of the scan | av-scan-start | No |
| AV_SIGNATURE_METADATA | The tag/metadata name representing file's AV type | av-signature | No |
| AV_STATUS_CLEAN | The value assigned to clean items inside of tags/metadata | CLEAN | No |
| AV_STATUS_INFECTED | The value assigned to clean items inside of tags/metadata | INFECTED | No |
| AV_STATUS_METADATA | The tag/metadata name representing file's AV status | av-status | No |
| AV_STATUS_SNS_ARN | SNS topic ARN to publish scan results (optional) | | No |
| AV_STATUS_SNS_PUBLISH_CLEAN | Publish AV_STATUS_CLEAN results to AV_STATUS_SNS_ARN | True | No |
| AV_STATUS_SNS_PUBLISH_INFECTED | Publish AV_STATUS_INFECTED results to AV_STATUS_SNS_ARN | True | No |
| AV_TIMESTAMP_METADATA | The tag/metadata name representing file's scan time | av-timestamp | No |
| CLAMAVLIB_PATH | Path to ClamAV library files | ./bin | No |
| CLAMSCAN_PATH | Path to ClamAV clamscan binary | ./bin/clamscan | No |
| FRESHCLAM_PATH | Path to ClamAV freshclam binary | ./bin/freshclam | No |
| DATADOG_API_KEY | API Key for pushing metrics to DataDog (optional) | | No |
| AV_PROCESS_ORIGINAL_VERSION_ONLY | Controls that only original version of an S3 key is processed (if bucket versioning is enabled) | False | No |
| AV_DELETE_INFECTED_FILES | Controls whether infected files should be automatically deleted | False | No |
| EVENT_SOURCE | The source of antivirus scan event "S3" or "SNS" (optional) | S3 | No |
| S3_ENDPOINT | The Endpoint to use when interacting wth S3 | None | No |
| SNS_ENDPOINT | The Endpoint to use when interacting wth SNS | None | No |
| LAMBDA_ENDPOINT | The Endpoint to use when interacting wth Lambda | None | No |
| Variable | Description | Default | Required |
|----------------------------------|-------------------------------------------------------------------------------------------------|------------------|----------|
| AV_DEFINITION_S3_BUCKET | Bucket containing antivirus definition files | | Yes |
| AV_DEFINITION_S3_PREFIX | Prefix for antivirus definition files | clamav_defs | No |
| AV_DEFINITION_PATH | Path containing virus definition files at runtime | /tmp/clamav_defs | No |
| AV_SCAN_START_SNS_ARN | SNS topic ARN to publish notification about start of scan | | No |
| AV_SCAN_START_METADATA | The tag/metadata indicating the start of the scan | av-scan-start | No |
| AV_SIGNATURE_METADATA | The tag/metadata name representing file's AV type | av-signature | No |
| AV_STATUS_CLEAN | The value assigned to clean items inside of tags/metadata | CLEAN | No |
| AV_STATUS_INFECTED | The value assigned to clean items inside of tags/metadata | INFECTED | No |
| AV_STATUS_METADATA | The tag/metadata name representing file's AV status | av-status | No |
| AV_STATUS_SNS_ARN | SNS topic ARN to publish scan results (optional) | | No |
| AV_STATUS_SNS_PUBLISH_CLEAN | Publish AV_STATUS_CLEAN results to AV_STATUS_SNS_ARN | True | No |
| AV_STATUS_SNS_PUBLISH_INFECTED | Publish AV_STATUS_INFECTED results to AV_STATUS_SNS_ARN | True | No |
| AV_TIMESTAMP_METADATA | The tag/metadata name representing file's scan time | av-timestamp | No |
| AV_EXTRA_VIRUS_DEFINITIONS | Uses fangfrisch for extra antivirus definitions | False | No |
| CLAMAVLIB_PATH | Path to ClamAV library files | ./bin | No |
| CLAMDSCAN_PATH | Path to ClamAV clamdscan binary | ./bin/clamdscan | No |
| FRESHCLAM_PATH | Path to ClamAV freshclam binary | ./bin/freshclam | No |
| DATADOG_API_KEY | API Key for pushing metrics to DataDog (optional) | | No |
| AV_PROCESS_ORIGINAL_VERSION_ONLY | Controls that only original version of an S3 key is processed (if bucket versioning is enabled) | False | No |
| AV_DELETE_INFECTED_FILES | Controls whether infected files should be automatically deleted | False | No |
| EVENT_SOURCE | The source of antivirus scan event "S3" or "SNS" (optional) | S3 | No |
| S3_ENDPOINT | The Endpoint to use when interacting wth S3 | None | No |
| SNS_ENDPOINT | The Endpoint to use when interacting wth SNS | None | No |
| LAMBDA_ENDPOINT | The Endpoint to use when interacting wth Lambda | None | No |

## S3 Bucket Policy Examples

Expand Down
Loading

0 comments on commit 4cf9df2

Please sign in to comment.