Merge pull request #100 from upsidetravel/cg_local_docker_development

Allow local testing of lambdas

.envrc

@@ -0,0 +1,77 @@
+#! /usr/bin/env bash
+
+##########################################
+# DO NOT MAKE LOCAL CHANGES TO THIS FILE #
+#                                        #
+# Vars in this file can be overridden by #
+# exporting them in .envrc.local         #
+##########################################
+
+# Add local paths for binaries and scripts
+PATH_add ./scripts
+
+# ShellCheck complains about things like `foo=$(cmd)` because you lose the
+# return value of `cmd`. That said, we're not using `set -e`, so we aren't
+# really concerned about return values. The following `true`, applies the
+# rule to the entire file.
+# See: https://github.com/koalaman/shellcheck/wiki/SC2155
+# shellcheck disable=SC2155
+true
+
+required_vars=()
+var_docs=()
+
+# Declare an environment variable as required.
+#
+#   require VAR_NAME "Documentation about how to define valid values"
+require() {
+  required_vars+=("$1")
+  var_docs+=("$2")
+}
+
+# Check all variables declared as required. If any are missing, print a message and
+# exit with a non-zero status.
+check_required_variables() {
+  for i in "${!required_vars[@]}"; do
+    var=${required_vars[i]}
+    if [[ -z "${!var}" ]]; then
+      log_status "${var} is not set: ${var_docs[i]}"
+      missing_var=true
+    fi
+  done
+
+  if [[ $missing_var == "true" ]]; then
+    log_error "Your environment is missing some variables!"
+    log_error "Set the above variables in .envrc.local and try again."
+  fi
+}
+
+#########################
+# Project Configuration #
+#########################
+
+# Lamdba resource constraints (Override in .envrc.local)
+# https://docs.docker.com/config/containers/resource_constraints/
+export MEM=1024m
+export CPUS=1.0
+
+require AV_DEFINITION_S3_BUCKET "Add this variable to your .envrc.local"
+require AV_DEFINITION_S3_PREFIX "Add this variable to your .envrc.local"
+
+require TEST_BUCKET "Add this variable to your .envrc.local"
+require TEST_KEY "Add this variable to your .envrc.local"
+
+##############################################
+# Load Local Overrides and Check Environment #
+##############################################
+
+# Load a local overrides file. Any changes you want to make for your local
+# environment should live in that file.
+
+if [ -e .envrc.local ]
+then
+  source_env .envrc.local
+fi
+
+# Check that all required environment variables are set
+check_required_variables

.envrc.local.template

@@ -0,0 +1,48 @@
+#! /usr/bin/env bash
+
+#
+# Copy this file `cp .envrc.local.template .envrc.local` and modify the variables below for testing
+#
+
+# Optional AWS Parameters
+# WARNING: It's not recommended to keep credentials in this file!
+# export AWS_ACCESS_KEY_ID
+# export AWS_DEFAULT_REGION
+# export AWS_REGION
+# export AWS_SECRET_ACCESS_KEY
+# export AWS_SESSION_TOKEN
+
+# Lamdba resource constraints you can override here
+# https://docs.docker.com/config/containers/resource_constraints/
+# export MEM=1024m
+# export CPUS=1.0
+
+# Required for both scan and update lambdas scripts
+export AV_DEFINITION_S3_BUCKET=""
+export AV_DEFINITION_S3_PREFIX=""
+
+# Required for scan lambda script
+export TEST_BUCKET=""
+export TEST_KEY=""
+
+# Uncomment and change as needed for lambda scripts
+# export AV_DEFINITION_FILE_PREFIXES
+# export AV_DEFINITION_FILE_SUFFIXES
+# export AV_DEFINITION_PATH
+# export AV_DELETE_INFECTED_FILES
+# export AV_PROCESS_ORIGINAL_VERSION_ONLY
+# export AV_SCAN_START_METADATA
+# export AV_SCAN_START_SNS_ARN
+# export AV_SIGNATURE_METADATA
+# export AV_SIGNATURE_OK
+# export AV_SIGNATURE_UNKNOWN
+# export AV_STATUS_CLEAN
+# export AV_STATUS_INFECTED
+# export AV_STATUS_METADATA
+# export AV_STATUS_SNS_ARN
+# export AV_STATUS_SNS_PUBLISH_CLEAN
+# export AV_STATUS_SNS_PUBLISH_INFECTED
+# export AV_TIMESTAMP_METADATA
+# export CLAMAVLIB_PATH
+# export CLAMSCAN_PATH
+# export FRESHCLAM_PATH

.gitignore

@@ -110,3 +110,10 @@ ENV/
 .coverage
 
 .DS_Store
+tmp/
+
+# direnv
+.envrc.local
+
+# EICAR Files
+*eicar*

Dockerfile

@@ -0,0 +1,45 @@
+FROM amazonlinux:2
+
+# Set up working directories
+RUN mkdir -p /opt/app
+RUN mkdir -p /opt/app/build
+RUN mkdir -p /opt/app/bin/
+
+# Copy in the lambda source
+WORKDIR /opt/app
+COPY ./*.py /opt/app/
+COPY requirements.txt /opt/app/requirements.txt
+
+# Install packages
+RUN yum update -y
+RUN yum install -y cpio python2-pip yum-utils zip unzip less
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+# This had --no-cache-dir, tracing through multiple tickets led to a problem in wheel
+RUN pip install -r requirements.txt
+RUN rm -rf /root/.cache/pip
+
+# Download libraries we need to run in lambda
+WORKDIR /tmp
+RUN yumdownloader -x \*i686 --archlist=x86_64 clamav clamav-lib clamav-update json-c pcre2
+RUN rpm2cpio clamav-0*.rpm | cpio -idmv
+RUN rpm2cpio clamav-lib*.rpm | cpio -idmv
+RUN rpm2cpio clamav-update*.rpm | cpio -idmv
+RUN rpm2cpio json-c*.rpm | cpio -idmv
+RUN rpm2cpio pcre*.rpm | cpio -idmv
+
+# Copy over the binaries and libraries
+RUN cp /tmp/usr/bin/clamscan /tmp/usr/bin/freshclam /tmp/usr/lib64/* /opt/app/bin/
+
+# Fix the freshclam.conf settings
+RUN echo "DatabaseMirror database.clamav.net" > /opt/app/bin/freshclam.conf
+RUN echo "CompressLocalDatabase yes" >> /opt/app/bin/freshclam.conf
+
+# Create the zip file
+WORKDIR /opt/app
+RUN zip -r9 --exclude="*test*" /opt/app/build/lambda.zip *.py bin
+
+WORKDIR /usr/lib/python2.7/site-packages
+RUN zip -r9 /opt/app/build/lambda.zip *
+
+WORKDIR /opt/app

Makefile

@@ -27,25 +27,16 @@ all: archive  ## Build the entire project
 clean:  ## Clean build artifacts
 	rm -rf bin/
 	rm -rf build/
+	rm -rf tmp/
 	rm -f .coverage
 	find ./ -type d -name '__pycache__' -delete
 	find ./ -type f -name '*.pyc' -delete
 
 .PHONY: archive
 archive: clean  ## Create the archive for AWS lambda
-ifeq ($(circleci), true)
-	docker create -v $(container_dir) --name src alpine:3.4 /bin/true
-	docker cp $(current_dir)/. src:$(container_dir)
-	docker run --rm -ti \
-		--volumes-from src \
-		amazonlinux:$(AMZ_LINUX_VERSION) \
-		/bin/bash -c "cd $(container_dir) && ./build_lambda.sh"
-else
-	docker run --rm -ti \
-		-v $(current_dir):$(container_dir) \
-		amazonlinux:$(AMZ_LINUX_VERSION) \
-		/bin/bash -c "cd $(container_dir) && ./build_lambda.sh"
-endif
+	docker build -t bucket-antivirus-function:latest .
+	mkdir -p ./build/
+	docker run -v $(current_dir)/build:/opt/mount --rm --entrypoint cp bucket-antivirus-function:latest /opt/app/build/lambda.zip /opt/mount/lambda.zip
 
 .PHONY: pre_commit_install  ## Ensure that pre-commit hook is installed and kept up to date
 pre_commit_install: .git/hooks/pre-commit ## Ensure pre-commit is installed
@@ -65,3 +56,11 @@ test: clean  ## Run python tests
 .PHONY: coverage
 coverage: clean  ## Run python tests with coverage
 	nosetests --with-coverage
+
+.PHONY: scan
+scan: ./build/lambda.zip ## Run scan function locally
+	scripts/run-scan-lambda $(TEST_BUCKET) $(TEST_KEY)
+
+.PHONY: update
+update: ./build/lambda.zip ## Run update function locally
+	scripts/run-update-lambda

README.md

@@ -133,7 +133,7 @@ and set its value to the name of the bucket created to store your AV
 definitions.
 11. Set *Lambda handler* to `update.lambda_handler`
 12. Under *Basic Settings*, set *Timeout* to **5 minutes** and *Memory* to
-**512**
+**1024**
 13. Save and test your function.  If prompted for test data, just use
 the default provided.
 
@@ -367,6 +367,31 @@ pip install -r requirements-dev.txt
 make test
 ```
 
+### Local lambdas
+
+You can run the lambdas locally to test out what they are doing without deploying to AWS. This is accomplished
+by using docker containers that act similarly to lambda. You will need to have set up some local variables in your
+`.envrc.local` file and modify them appropriately first before running `direnv allow`. If you do not have `direnv`
+it can be installed with `brew install direnv`.
+
+For the Scan lambda you will need a test file uploaded to S3 and the variables `TEST_BUCKET` and `TEST_KEY`
+set in your `.envrc.local` file. Then you can run:
+
+```sh
+direnv allow
+make archive scan
+```
+
+If you want a file that will be recognized as a virus you can download a test file from the [EICAR](https://www.eicar.org/?page_id=3950)
+website and uploaded to your bucket.
+
+For the Update lambda you can run:
+
+```sh
+direnv allow
+make archive update
+```
+
 ## License
 
 ```text

clamav.py

@@ -146,7 +146,12 @@ def md5_from_s3_tags(s3_client, bucket, key):
     try:
         tags = s3_client.get_object_tagging(Bucket=bucket, Key=key)["TagSet"]
     except botocore.exceptions.ClientError as e:
-        expected_errors = {"404", "AccessDenied", "NoSuchKey"}
+        expected_errors = {
+            "404",  # Object does not exist
+            "AccessDenied",  # Object cannot be accessed
+            "NoSuchKey",  # Object does not exist
+            "MethodNotAllowed",  # Object deleted in bucket with versioning
+        }
         if e.response["Error"]["Code"] in expected_errors:
             return ""
         else:

scripts/run-scan-lambda

@@ -0,0 +1,52 @@
+#! /usr/bin/env bash
+
+set -eu -o pipefail
+
+#
+# Run the scan.lambda_handler locally in a docker container
+#
+
+if [ $# -lt 2 ]; then
+  echo 1>&2 "$0: not enough arguments. Please provide BUCKET and KEY"
+  exit 1
+fi
+
+BUCKET=$1
+KEY=$2
+EVENT="{\"Records\": [{\"s3\": {\"bucket\": {\"name\": \"${BUCKET}\"}, \"object\": {\"key\": \"${KEY}\"}}}]}"
+echo "Sending S3 event: ${EVENT}"
+
+# Verify that the file exists first
+aws s3 ls "s3://${BUCKET}/${KEY}"
+
+rm -rf tmp/
+unzip -qq -d ./tmp build/lambda.zip
+
+NAME="antivirus-scan"
+
+docker run --rm \
+  -v "$(pwd)/tmp/:/var/task" \
+  -e AV_DEFINITION_S3_BUCKET \
+  -e AV_DEFINITION_S3_PREFIX \
+  -e AV_DELETE_INFECTED_FILES \
+  -e AV_PROCESS_ORIGINAL_VERSION_ONLY \
+  -e AV_SCAN_START_METADATA \
+  -e AV_SCAN_START_SNS_ARN \
+  -e AV_SIGNATURE_METADATA \
+  -e AV_STATUS_CLEAN \
+  -e AV_STATUS_INFECTED \
+  -e AV_STATUS_METADATA \
+  -e AV_STATUS_SNS_ARN \
+  -e AV_STATUS_SNS_PUBLISH_CLEAN \
+  -e AV_STATUS_SNS_PUBLISH_INFECTED \
+  -e AV_TIMESTAMP_METADATA \
+  -e AWS_ACCESS_KEY_ID \
+  -e AWS_DEFAULT_REGION \
+  -e AWS_REGION \
+  -e AWS_SECRET_ACCESS_KEY \
+  -e AWS_SESSION_TOKEN \
+  --memory="${MEM}" \
+  --memory-swap="${MEM}" \
+  --cpus="${CPUS}" \
+  --name="${NAME}" \
+  lambci/lambda:python2.7 scan.lambda_handler "${EVENT}"