Skip to content
This repository has been archived by the owner on Jun 20, 2023. It is now read-only.

Reduce warm lambda processing time and add SAM framework #112

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Reduce warm lambda processing time and add SAM framework #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .envrc.local.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,5 @@ export TEST_KEY=""
# export AV_STATUS_SNS_PUBLISH_INFECTED
# export AV_TIMESTAMP_METADATA
# export CLAMAVLIB_PATH
# export CLAMSCAN_PATH
# export CLAMDSCAN_PATH
# export FRESHCLAM_PATH
5 changes: 5 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,8 @@ tmp/

# EICAR Files
*eicar*

# aws-sam
.aws-sam
samconfig.toml
.sam-env
9 changes: 9 additions & 0 deletions .sam-env.template
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"ScanFunction": {
"AV_DEFINITION_S3_BUCKET": "",
"AV_STATUS_SNS_ARN": ""
},
"UpdateFunction": {
"AV_DEFINITION_S3_BUCKET": ""
}
}
57 changes: 30 additions & 27 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,48 @@
FROM amazonlinux:2

# Set up working directories
RUN mkdir -p /opt/app
RUN mkdir -p /opt/app/build
RUN mkdir -p /opt/app/bin/

# Copy in the lambda source
WORKDIR /opt/app
COPY ./*.py /opt/app/
COPY requirements.txt /opt/app/requirements.txt
RUN mkdir -p /opt/app/clamav

# Install packages
RUN yum update -y
RUN yum install -y cpio python3-pip yum-utils zip unzip less
RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm

# This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
RUN pip3 install -r requirements.txt
RUN rm -rf /root/.cache/pip
RUN amazon-linux-extras install epel -y
RUN yum install -y cpio yum-utils zip

# Download libraries we need to run in lambda
WORKDIR /tmp
RUN yumdownloader -x \*i686 --archlist=x86_64 clamav clamav-lib clamav-update json-c pcre2
RUN yumdownloader -x \*i686 --archlist=x86_64 \
clamav \
clamav-lib \
clamav-scanner-systemd \
clamav-update \
elfutils-libs \
json-c \
lz4 \
pcre2 \
systemd-libs

RUN rpm2cpio clamav-0*.rpm | cpio -idmv
RUN rpm2cpio clamav-lib*.rpm | cpio -idmv
RUN rpm2cpio clamav-update*.rpm | cpio -idmv
RUN rpm2cpio clamd-0*.rpm | cpio -idmv
RUN rpm2cpio elfutils-libs*.rpm | cpio -idmv
RUN rpm2cpio json-c*.rpm | cpio -idmv
RUN rpm2cpio lz4*.rpm | cpio -idmv
RUN rpm2cpio pcre*.rpm | cpio -idmv
RUN rpm2cpio systemd-libs*.rpm | cpio -idmv

# Copy over the binaries and libraries
RUN cp /tmp/usr/bin/clamscan /tmp/usr/bin/freshclam /tmp/usr/lib64/* /opt/app/bin/
RUN cp -r /tmp/usr/bin/clamdscan \
/tmp/usr/sbin/clamd \
/tmp/usr/bin/freshclam \
/tmp/usr/lib64/* \
/opt/app/clamav/

# Fix the freshclam.conf settings
RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf
RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
RUN echo "DatabaseDirectory /tmp/clamav_defs" > /opt/app/clamav/scan.conf
RUN echo "PidFile /tmp/clamd.pid" >> /opt/app/clamav/scan.conf
RUN echo "LocalSocket /tmp/clamd.sock" >> /opt/app/clamav/scan.conf
RUN echo "LogFile /tmp/clamd.log" >> /opt/app/clamav/scan.conf

# Create the zip file
WORKDIR /opt/app
RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin

WORKDIR /usr/local/lib/python3.7/site-packages
RUN zip -r9 /opt/app/build/lambda.zip *

WORKDIR /opt/app
# Fix the freshclam.conf settings
RUN echo "DatabaseMirror database.clamav.net" > /opt/app/clamav/freshclam.conf
RUN echo "CompressLocalDatabase yes" >> /opt/app/clamav/freshclam.conf
19 changes: 11 additions & 8 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

AMZ_LINUX_VERSION:=2
current_dir := $(shell pwd)
container_dir := /opt/app
container_dir := /opt/app/clamav
circleci := ${CIRCLECI}

.PHONY: help
Expand All @@ -26,7 +26,6 @@ all: archive ## Build the entire project
.PHONY: clean
clean: ## Clean build artifacts
rm -rf bin/
rm -rf build/
rm -rf tmp/
rm -f .coverage
find ./ -type d -name '__pycache__' -delete
Expand All @@ -35,8 +34,9 @@ clean: ## Clean build artifacts
.PHONY: archive
archive: clean ## Create the archive for AWS lambda
docker build -t bucket-antivirus-function:latest .
mkdir -p ./build/
docker run -v $(current_dir)/build:/opt/mount --rm --entrypoint cp bucket-antivirus-function:latest /opt/app/build/lambda.zip /opt/mount/lambda.zip
mkdir -p ./bin
docker run -v $(current_dir)/bin:/opt/mount --rm bucket-antivirus-function:latest bash -c "cp -r ${container_dir}/* /opt/mount/"
sam build --use-container

.PHONY: pre_commit_install ## Ensure that pre-commit hook is installed and kept up to date
pre_commit_install: .git/hooks/pre-commit ## Ensure pre-commit is installed
Expand All @@ -58,9 +58,12 @@ coverage: clean ## Run python tests with coverage
nosetests --with-coverage

.PHONY: scan
scan: ./build/lambda.zip ## Run scan function locally
scripts/run-scan-lambda $(TEST_BUCKET) $(TEST_KEY)
scan:
sam local generate-event s3 put \
--bucket $(TEST_BUCKET) \
--key $(TEST_KEY) \
| sam local invoke ScanFunction --env-vars .sam-env -e -

.PHONY: update
update: ./build/lambda.zip ## Run update function locally
scripts/run-update-lambda
update:
sam local invoke UpdateFunction --env-vars .sam-env --no-event
205 changes: 5 additions & 200 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,207 +34,12 @@ or INFECTED, along with the date and time of the scan.

### Build from Source

To build the archive to upload to AWS Lambda, run `make`. The build process is completed using
the [amazonlinux](https://hub.docker.com/_/amazonlinux/) [Docker](https://www.docker.com)
image. The resulting archive will be built at `build/lambda.zip`. This file will be
uploaded to AWS for both Lambda functions below.
To build all functions and clamav binaries, run `make`. The build process is completed using
the [Serverless Application Model](https://aws.amazon.com/serverless/sam/) framework.

### AV Definition Bucket
### Deployment

Create an s3 bucket to store current antivirus definitions. This
provides the fastest download speeds for the scanner. This bucket can
be kept as private.

To allow public access, useful for other accounts,
add the following policy to the bucket.

```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowPublic",
"Effect": "Allow",
"Principal": "*",
"Action": [
"s3:GetObject",
"s3:GetObjectTagging"
],
"Resource": "arn:aws:s3:::<bucket-name>/*"
}
]
}
```

### Definition Update Lambda

This function accesses the user’s ClamAV instance to download
updated definitions using `freshclam`. It is recommended to run
this every 3 hours to stay protected from the latest threats.

1. Create the archive using the method in the
[Build from Source](#build-from-source) section.
2. From the AWS Lambda Dashboard, click **Create function**
3. Choose **Author from scratch** on the *Create function* page
4. Name your function `bucket-antivirus-update` when prompted on the
*Configure function* step.
5. Set *Runtime* to `Python 3.7`
6. Create a new role name `bucket-antivirus-update` that uses the
following policy document

```json
{
"Version":"2012-10-17",
"Statement":[
{
"Sid":"WriteCloudWatchLogs",
"Effect":"Allow",
"Action":[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":"*"
},
{
"Sid":"s3GetAndPutWithTagging",
"Action":[
"s3:GetObject",
"s3:GetObjectTagging",
"s3:PutObject",
"s3:PutObjectTagging",
"s3:PutObjectVersionTagging"
],
"Effect":"Allow",
"Resource":[
"arn:aws:s3:::<av-definition-s3-bucket>/*"
]
},
{
"Sid": "s3HeadObject",
"Effect": "Allow",
"Action": "s3:ListBucket",
"Resource": [
"arn:aws:s3:::<av-definition-s3-bucket>/*",
"arn:aws:s3:::<av-definition-s3-bucket>"
]
}
]
}
```

7. Click next to go to the Configuration page
8. Add a trigger from the left of **CloudWatch Event** using `rate(3 hours)`
for the **Schedule expression**. Be sure to check **Enable trigger**
9. Choose **Upload a ZIP file** for *Code entry type* and select the archive
downloaded in step 1.
10. Add a single environment variable named `AV_DEFINITION_S3_BUCKET`
and set its value to the name of the bucket created to store your AV
definitions.
11. Set *Lambda handler* to `update.lambda_handler`
12. Under *Basic Settings*, set *Timeout* to **5 minutes** and *Memory* to
**1024**
13. Save and test your function. If prompted for test data, just use
the default provided.

### AV Scanner Lambda

1. Create the archive using the method in the
[Build from Source](#build-from-source) section.
2. From the AWS Lambda Dashboard, click **Create function**
3. Choose **Author from scratch** on the *Create function* page
4. Name your function `bucket-antivirus-function`
5. Set *Runtime* to `Python 3.7`
6. Create a new role name `bucket-antivirus-function` that uses the
following policy document

```json
{
"Version":"2012-10-17",
"Statement":[
{
"Sid":"WriteCloudWatchLogs",
"Effect":"Allow",
"Action":[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":"*"
},
{
"Sid":"s3AntiVirusScan",
"Action":[
"s3:GetObject",
"s3:GetObjectTagging",
"s3:GetObjectVersion",
"s3:PutObjectTagging",
"s3:PutObjectVersionTagging"
],
"Effect":"Allow",
"Resource": [
"arn:aws:s3:::<bucket-name-1>/*",
"arn:aws:s3:::<bucket-name-2>/*"
]
},
{
"Sid":"s3AntiVirusDefinitions",
"Action":[
"s3:GetObject",
"s3:GetObjectTagging"
],
"Effect":"Allow",
"Resource": [
"arn:aws:s3:::<av-definition-s3-bucket>/*"
]
},
{
"Sid":"kmsDecrypt",
"Action":[
"kms:Decrypt"
],
"Effect":"Allow",
"Resource": [
"arn:aws:s3:::<bucket-name-1>/*",
"arn:aws:s3:::<bucket-name-2>/*"
]
},
{
"Sid":"snsPublish",
"Action": [
"sns:Publish"
],
"Effect":"Allow",
"Resource": [
"arn:aws:sns:::<av-scan-start>",
"arn:aws:sns:::<av-status>"
]
},
{
"Sid":"s3HeadObject",
"Effect":"Allow",
"Action":"s3:ListBucket",
"Resource":[
"arn:aws:s3:::<av-definition-s3-bucket>/*",
"arn:aws:s3:::<av-definition-s3-bucket>"
]
}
]
}
```

7. Click *next* to head to the Configuration page
8. Add a new trigger of type **S3 Event** using `ObjectCreate(all)`.
9. Choose **Upload a ZIP file** for *Code entry type* and select the archive
created in step 1.
10. Set *Lambda handler* to `scan.lambda_handler`
11. Add a single environment variable named `AV_DEFINITION_S3_BUCKET`
and set its value to the name of the bucket created to store your AV
definitions. If your bucket is `s3://my-bucket`, the value should be `my-bucket`.
12. Under *Basic settings*, set *Timeout* to **5 minutes** and *Memory* to
**1024**
13. Save the function. Testing is easiest performed by uploading a
file to the bucket configured as the trigger in step 4.
Use `sam deploy --guided` to deploy the stack to AWS. The guided deployment will require a list of buckets the function can scan. They should be provided in the following format: `arn:aws:s3:::<BUCKET_NAME>/*`

### S3 Events

Expand Down Expand Up @@ -270,7 +75,7 @@ the table below for reference.
| AV_STATUS_SNS_PUBLISH_INFECTED | Publish AV_STATUS_INFECTED results to AV_STATUS_SNS_ARN | True | No |
| AV_TIMESTAMP_METADATA | The tag/metadata name representing file's scan time | av-timestamp | No |
| CLAMAVLIB_PATH | Path to ClamAV library files | ./bin | No |
| CLAMSCAN_PATH | Path to ClamAV clamscan binary | ./bin/clamscan | No |
| CLAMDSCAN_PATH | Path to ClamAV clamdscan binary | ./bin/clamdscan | No |
| FRESHCLAM_PATH | Path to ClamAV freshclam binary | ./bin/freshclam | No |
| DATADOG_API_KEY | API Key for pushing metrics to DataDog (optional) | | No |
| AV_PROCESS_ORIGINAL_VERSION_ONLY | Controls that only original version of an S3 key is processed (if bucket versioning is enabled) | False | No |
Expand Down
Loading