Add support for:

.circleci/config.yml

@@ -15,10 +15,10 @@ executors:
       - image: docker:17.05.0-ce
   python_test:
     docker:
-      - image: circleci/python:3.7-stretch
+      - image: circleci/python:3.8
   pre_commit_test:
     docker:
-      - image: circleci/python:3.7-stretch
+      - image: circleci/python:3.8
 
 jobs:
 
@@ -93,4 +93,4 @@ workflows:
       - build:
           requires:
             - pre_commit_test
-            - test
+            - test

Dockerfile

@@ -11,40 +11,58 @@ COPY ./*.py /opt/app/
 COPY requirements.txt /opt/app/requirements.txt
 
 # Install packages
-RUN yum update -y
-RUN yum install -y cpio python3-pip yum-utils zip unzip less
-RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum update -y && \
+    yum groupinstall -y "Development Tools" && \
+    yum install -y yum-utils cpio zip unzip less wget && \
+    yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 
-# This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
-RUN pip3 install -r requirements.txt
-RUN rm -rf /root/.cache/pip
+# Install amazon-linux-extras in order to install python3.8
+RUN yum install -y amazon-linux-extras && \
+    amazon-linux-extras enable python3.8 && \
+    yum -y install python3.8 && \
+    python3.8 -m pip install -r requirements.txt && \
+    # This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
+    rm -rf /root/.cache/pip && \
+    python3.8 -m pip install -U pytest
 
-# Download libraries we need to run in lambda
+# Download libraries we need to run in lambda with python3.8
 WORKDIR /tmp
-RUN yumdownloader -x \*i686 --archlist=x86_64 clamav clamav-lib clamav-update json-c pcre2 libprelude gnutls libtasn1 lib64nettle nettle
-RUN rpm2cpio clamav-0*.rpm | cpio -idmv
-RUN rpm2cpio clamav-lib*.rpm | cpio -idmv
-RUN rpm2cpio clamav-update*.rpm | cpio -idmv
-RUN rpm2cpio json-c*.rpm | cpio -idmv
-RUN rpm2cpio pcre*.rpm | cpio -idmv
-RUN rpm2cpio gnutls* | cpio -idmv
-RUN rpm2cpio nettle* | cpio -idmv
-RUN rpm2cpio lib* | cpio -idmv
-RUN rpm2cpio *.rpm | cpio -idmv
-RUN rpm2cpio libtasn1* | cpio -idmv
+RUN wget https://www.clamav.net/downloads/production/clamav-0.104.0.linux.x86_64.rpm && \
+    yumdownloader -x \*i686 --archlist=x86_64 \
+          json-c pcre2 libprelude gnutls libtasn1 lib64nettle nettle \
+          bzip2-libs libtool-ltdl libxml2 xz-libs
+
+RUN \
+    rpm2cpio clamav-0*.rpm | cpio -idmv && \
+    rpm2cpio json-c*.rpm | cpio -idmv && \
+    rpm2cpio pcre*.rpm | cpio -idmv && \
+    rpm2cpio gnutls* | cpio -idmv && \
+    rpm2cpio nettle* | cpio -idmv && \
+    rpm2cpio lib* | cpio -idmv && \
+    rpm2cpio *.rpm | cpio -idmv && \
+    rpm2cpio libtasn1* | cpio -idmv && \
+    rpm2cpio bzip2-libs*.rpm | cpio -idmv && \
+    rpm2cpio libtool-ltdl*.rpm | cpio -idmv && \
+    rpm2cpio libxml2*.rpm | cpio -idmv && \
+    rpm2cpio xz-libs*.rpm | cpio -idmv
 
 # Copy over the binaries and libraries
-RUN cp /tmp/usr/bin/clamscan /tmp/usr/bin/freshclam /tmp/usr/lib64/* /opt/app/bin/
+RUN cp /tmp/usr/lib64/* \
+       /tmp/usr/local/bin/clamscan \
+       /tmp/usr/local/bin/freshclam \
+       /tmp/usr/local/lib64/libclam* \
+       /opt/app/bin/
 
 # Fix the freshclam.conf settings
-RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf
-RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
+RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf && \
+    echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
 
 # Create the zip file
 WORKDIR /opt/app
 RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
 
-WORKDIR /usr/local/lib/python3.7/site-packages
+# Change path to Python 3.8
+WORKDIR /usr/local/lib/python3.8/site-packages
 RUN zip -r9 /opt/app/build/lambda.zip *
 
 WORKDIR /opt/app

Dockerfile.aarch64

@@ -0,0 +1,132 @@
+FROM amazonlinux:2
+
+# Set up working directories
+RUN mkdir -p /opt/app
+RUN mkdir -p /opt/app/build
+RUN mkdir -p /opt/app/bin/
+
+# Copy in the lambda source
+WORKDIR /opt/app
+COPY ./*.py /opt/app/
+COPY requirements.txt /opt/app/requirements.txt
+
+# Install packages
+RUN yum update -y && \
+    yum groupinstall -y "Development Tools" && \
+    yum install -y yum-utils cpio zip unzip less wget && \
+    yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+# Install amazon-linux-extras in order to install python3.8
+RUN yum install -y amazon-linux-extras && \
+    amazon-linux-extras enable python3.8 && \
+    yum -y install python3.8 && \
+    python3.8 -m pip install -r requirements.txt && \
+    # This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
+    rm -rf /root/.cache/pip && \
+    python3.8 -m pip install -U pytest
+
+# Install additional packages for local compilation of cmake and clamav for aarch64
+RUN yum install -y gcc gcc-c++ make valgrind openssl-devel \
+     libxml2 bzip2-devel json-c-devel libffi-devel \
+     check-devel  libxml2-devel libxslt-devel pcre2-devel \
+     zlib-devel libcurl-devel ncurses-devel sendmail-devel
+
+
+# Download libraries we need to run in lambda with python3.8
+WORKDIR /tmp
+RUN yumdownloader -x \*i686 --archlist=aarch64 \
+          json-c pcre2 libprelude gnutls libtasn1 lib64nettle nettle \
+          bzip2-libs libtool-ltdl libxml2 xz-libs \
+          # Additional packages \
+          libgpg-error libcurl libnghttp2 libidn2
+
+RUN \
+  rpm2cpio json-c*.rpm | cpio -idmv && \
+  rpm2cpio pcre*.rpm | cpio -idmv && \
+  rpm2cpio gnutls* | cpio -idmv && \
+  rpm2cpio nettle* | cpio -idmv && \
+  rpm2cpio lib* | cpio -idmv && \
+  rpm2cpio *.rpm | cpio -idmv && \
+  rpm2cpio libtasn1* | cpio -idmv && \
+  rpm2cpio bzip2-libs*.rpm | cpio -idmv && \
+  rpm2cpio libtool-ltdl*.rpm | cpio -idmv && \
+  rpm2cpio libxml2*.rpm | cpio -idmv && \
+  rpm2cpio xz-libs*.rpm | cpio -idmv && \
+  # Additional packages \
+  rpm2cpio libgpg-error*.rpm | cpio -idmv && \
+  rpm2cpio libcurl*.rpm | cpio -idmv && \
+  rpm2cpio libnghttp2*.rpm | cpio -idmv && \
+  rpm2cpio libidn2*.rpm | cpio -idmv
+
+RUN cp /tmp/usr/lib64/* \
+       # libcrypt* extracts into /tmp/lib64 and not /tmp/usr/lib64 \
+       /tmp/lib64/*  \
+       # Copy over other additional dependencies \
+       /usr/lib64/libldap-2.4* \
+       /usr/lib64/libssh2* \
+       /usr/lib64/liblber* \
+       /usr/lib64/libsmime3* \
+       /usr/lib64/libunistring* \
+       /usr/lib64/libsasl2* \
+       /usr/lib64/libssl* \
+       /usr/lib64/libssl* \
+       /usr/lib64/libnss3* \
+       /usr/lib64/libgssapi* \
+       /usr/lib64/libkrb5* \
+       /usr/lib64/libk5crypto* \
+       /usr/lib64/libkrb5support* \
+       /usr/lib64/libkeyutils*  \
+       /usr/lib64/libpl* \
+       /usr/lib64/libnspr* \
+       /usr/lib64/libdl* \
+       /usr/lib64/libresolv* \
+       /usr/lib64/libnssutil* \
+       /usr/lib64/libcrypt* \
+       /opt/app/bin
+
+# Build cmake v3.21 locally as the version available is not compatible for building clamav 0.104+
+WORKDIR /tmp
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.21.2/cmake-3.21.2-linux-aarch64.sh && \
+  yum remove cmake && \
+  echo 'y' | sh cmake-3.21.2-linux-aarch64.sh && \
+  cp cmake-3.21*/bin/c* /usr/local/bin/ && \
+  cp -r cmake-3.21*/share/* /usr/local/share/
+
+# Build clamav v0.104 locally using cmake
+WORKDIR /tmp
+RUN wget https://www.clamav.net/downloads/production/clamav-0.104.0.tar.gz && \
+    gunzip clamav-0.*tar.gz && \
+    tar xvf clamav-0.*tar && \
+    cd clamav* && \
+    mkdir build && \
+    cd build && \
+    cmake .. \
+      -D CMAKE_INSTALL_PREFIX=/usr \
+      -D CMAKE_INSTALL_LIBDIR=lib \
+      -D APP_CONFIG_DIRECTORY=/etc/clamav \
+      -D DATABASE_DIRECTORY=/var/lib/clamav \
+      -D ENABLE_JSON_SHARED=ON && \
+    cmake --build . && \
+    cmake --build . --target install && \
+    echo "Done building clamav"
+
+# Copy over the binaries and libraries
+RUN cp /usr/bin/clamscan \
+       /usr/bin/freshclam \
+       /usr/lib/*clam* \
+       /opt/app/bin
+
+
+# Fix the freshclam.conf settings
+RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf && \
+    echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
+
+# Create the zip file
+WORKDIR /opt/app
+RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
+
+# Change path to Python 3.8
+WORKDIR /usr/local/lib/python3.8/site-packages
+RUN zip -r9 /opt/app/build/lambda.zip *
+
+WORKDIR /opt/app

Makefile

@@ -17,6 +17,14 @@ current_dir := $(shell pwd)
 container_dir := /opt/app
 circleci := ${CIRCLECI}
 
+# use default Dockerfile for x86_64
+ARCH:=$(shell uname -i)
+DOCKERFILE := Dockerfile
+ifeq (aarch64,$(strip $(ARCH)))
+  # Use separate docker file for ARM arch
+  DOCKERFILE := 'Dockerfile.aarch64'
+endif
+
 .PHONY: help
 help:  ## Print the help documentation
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -34,7 +42,7 @@ clean:  ## Clean build artifacts
 
 .PHONY: archive
 archive: clean  ## Create the archive for AWS lambda
-	docker build -t bucket-antivirus-function:latest .
+	docker build -t bucket-antivirus-function:latest . -f $(DOCKERFILE)
 	mkdir -p ./build/
 	docker run -v $(current_dir)/build:/opt/mount --rm --entrypoint cp bucket-antivirus-function:latest /opt/app/build/lambda.zip /opt/mount/lambda.zip
 
@@ -63,4 +71,4 @@ scan: ./build/lambda.zip ## Run scan function locally
 
 .PHONY: update
 update: ./build/lambda.zip ## Run update function locally
-	scripts/run-update-lambda
+	scripts/run-update-lambda

deploy/cloudformation.yaml

@@ -200,7 +200,7 @@
       Properties:
         FunctionName: avUpdateDefinitions
         Description: LambdaFunction to update the AntiVirus definitions in the AV Definitions bucket.
-        Runtime: python3.7
+        Runtime: python3.8
         Code:
           ZipFile: |
             import json
@@ -247,7 +247,7 @@
       Properties:
         FunctionName: avScanner
         Description: LambdaFunction to scan newly uploaded objects in S3.
-        Runtime: python3.7
+        Runtime: python3.8
         Code:
           ZipFile: |
             import json
@@ -284,4 +284,4 @@
 
     IamRoleAVScanner:
       Value: !Ref IamRoleAVScanner
-      Description: IAM Role used by the Lambda Scanner function. Edit its policy to add/change source S3 buckets, and also to enable SNS functionality if desired
+      Description: IAM Role used by the Lambda Scanner function. Edit its policy to add/change source S3 buckets, and also to enable SNS functionality if desired

scripts/run-scan-lambda

@@ -49,4 +49,4 @@ docker run --rm \
   --memory-swap="${MEM}" \
   --cpus="${CPUS}" \
   --name="${NAME}" \
-  lambci/lambda:python3.7 scan.lambda_handler "${EVENT}"
+  lambci/lambda:python3.8 scan.lambda_handler "${EVENT}"

scripts/run-update-lambda

@@ -26,4 +26,4 @@ docker run --rm \
   --memory-swap="${MEM}" \
   --cpus="${CPUS}" \
   --name="${NAME}" \
-  lambci/lambda:python3.7 update.lambda_handler
+  lambci/lambda:python3.8 update.lambda_handler