Run end to end tests in their own container (#51)

Adds a bespoke end to end test container which depends on the successful completion of azurite setup. This means the azurite blob containers are present and writable.
The e2e test container shares logs with the function apps which allows it to poll for expected log output indicating the functions have been triggered and the execution was successful.

Author: steventux

compose.yml

@@ -42,6 +42,32 @@ services:
     profiles:
       - test
 
+  end-to-end-tests:
+    container_name: end-to-end-tests
+    network_mode: host
+    build:
+      context: ./tests/end_to_end/
+      additional_contexts:
+        root_dir: .
+      dockerfile: Dockerfile
+    environment:
+      - AZURITE_CONNECTION_STRING=${AZURITE_CONNECTION_STRING}
+      - BLOB_CONTAINER_NAME=${BLOB_CONTAINER_NAME}
+    profiles:
+      - test
+    depends_on:
+      azurite-setup:
+        condition: service_completed_successfully
+      azurite:
+        condition: service_started
+      notify:
+        condition: service_started
+      process-pilot-data:
+        condition: service_started
+    volumes:
+      - ./log/functions/notify:/tests/end_to_end/log/functions/notify
+      - ./log/functions/process-pilot-data:/tests/end_to_end/log/functions/process-pilot-data
+
   # Functions
   message-status:
     container_name: message-status
@@ -71,6 +97,8 @@ services:
       - OAUTH2_API_KID=${OAUTH2_API_KID}
       - OAUTH2_TOKEN_URL=${OAUTH2_TOKEN_URL}
       - PRIVATE_KEY=${PRIVATE_KEY}
+    volumes:
+      - ./log/functions/notify:/tmp/Functions/Host
 
   process-pilot-data:
     container_name: process-pilot-data
@@ -87,6 +115,8 @@ services:
       - BLOB_CONTAINER_NAME=${BLOB_CONTAINER_NAME}
       - CONTACT_TELEPHONE_NUMBER=${CONTACT_TELEPHONE_NUMBER}
       - NOTIFY_FUNCTION_URL=${NOTIFY_FUNCTION_URL}
+    volumes:
+      - ./log/functions/process-pilot-data:/tmp/Functions/Host
 
 networks:
   frontend:

src/functions/notify/host.json

@@ -3,6 +3,15 @@
   "extensionBundle": {
     "id": "Microsoft.Azure.Functions.ExtensionBundle",
     "version": "[4.*, 5.0.0)"
+  },
+  "logging": {
+    "fileLoggingMode": "always",
+    "logLevel": {
+      "default": "Information",
+      "Host.Results": "Error",
+      "Function": "Trace",
+      "Host.Aggregator": "Trace"
+    }
   }
 }
 

src/functions/process_pilot_data/host.json

@@ -3,6 +3,15 @@
   "extensionBundle": {
     "id": "Microsoft.Azure.Functions.ExtensionBundle",
     "version": "[4.*, 5.0.0)"
+  },
+  "logging": {
+    "fileLoggingMode": "always",
+    "logLevel": {
+      "default": "Information",
+      "Host.Results": "Error",
+      "Function": "Trace",
+      "Host.Aggregator": "Trace"
+    }
   }
 }
 

test-end-to-end.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+docker compose --env-file .env.e2e --profile test run --build --quiet-pull end-to-end-tests pytest /tests/end_to_end
+test_exit_code=$?
+docker compose --env-file .env.e2e --profile test down --volumes --remove-orphans
+exit $test_exit_code

test.sh

@@ -126,10 +126,17 @@ install_pipenv_dependencies() {
 
 run_all_test_suites() {
     tests_dir="tests/"
-    pytest $tests_dir || {
+    end_to_end_tests_dir="$tests_dir/end_to_end"
+    end_to_end_tests_script="./test-end-to-end.sh"
+
+    pytest --ignore=$end_to_end_tests_dir $tests_dir || {
         echo "Tests failed in $tests_dir"
         exit 1
     }
+    source $end_to_end_tests_script || {
+        echo "End to end tests failed in $end_to_end_tests_dir"
+        exit 1
+    }
 }
 
 # Actually run the things

tests/end_to_end/Dockerfile

@@ -0,0 +1,17 @@
+# To enable ssh & remote debugging on app service change the base image to the one below
+# FROM mcr.microsoft.com/azure-functions/python:4-python3.11-appservice
+FROM mcr.microsoft.com/azure-functions/python:4-python3.11
+
+ENV DATABASE_HOST=${DATABASE_HOST} \
+    DATABASE_USER=${DATABASE_USER} \
+    DATABASE_PASSWORD=${DATABASE_PASSWORD} \
+    DATABASE_NAME=${DATABASE_NAME}
+
+
+COPY --from=root_dir Pipfile /
+COPY --from=root_dir Pipfile.lock /
+RUN pip install pipenv
+RUN pipenv install --system
+RUN pipenv install --dev --system
+
+COPY . /tests/end_to_end

tests/test_end_to_end.py renamed to tests/end_to_end/test_file_upload_to_notifying_recipients.py

@@ -1,82 +1,19 @@
 import azure.storage.blob
 import dotenv
+import glob
 import logging
 import os
 import pytest
-import subprocess
 import time
 
-"""
-End to end test for the azure functions using docker compose.
-To run the test with logging output use the following command:
-
-pytest --log-cli-level=INFO tests/test_end_to_end.py
-"""
-
 ENV_FILE = os.getenv("ENV_FILE", ".env.e2e")
 
 
 @pytest.fixture()
 def setup():
-    if not os.getenv("GITHUB_ACTIONS"):
-        pytest.skip("Skipping end to end test, set GITHUB_ACTIONS env var to run this test")
-
     dotenv.load_dotenv(dotenv_path=ENV_FILE)
 
 
-@pytest.fixture
-def docker():
-    try:
-        assert docker_compose_build(), "Error building containers"
-        assert docker_compose_up()
-        yield
-    finally:
-        docker_compose_down()
-
-
-def docker_arglist(command, *args):
-    return ['docker', 'compose', '--env-file', ENV_FILE, '--profile', 'test', command, *args]
-
-
-def docker_compose_build():
-    logging.info("Building containers")
-    try:
-        subprocess.run(
-            docker_arglist('build', 'azurite', 'azurite-setup', 'notify', 'process-pilot-data'),
-            check=True,
-            stderr=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            text=True,
-        )
-        return True
-    except Exception as e:
-        logging.error(f"Error building containers: {e.stderr}")
-        return False
-
-
-def docker_compose_up():
-    logging.info("Starting containers")
-    try:
-        subprocess.Popen(
-            docker_arglist('up', '-d'),
-            stderr=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-        )
-        return True
-    except Exception as e:
-        logging.error(f"Error building containers: {e.stderr}")
-        return False
-
-
-def docker_compose_down():
-    logging.info("Stopping containers")
-    subprocess.run(
-        docker_arglist('down'),
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-
 def upload_file_to_blob_storage():
     logging.info("Uploading file to blob storage")
     try:
@@ -99,17 +36,17 @@ def upload_file_to_blob_storage():
     return True
 
 
+def log_dir_for_container(container_name):
+    return f"/tests/end_to_end/log/functions/{container_name}"
+
+
 def logs_for_container(container_name):
-    result = subprocess.run(
-        docker_arglist('logs', container_name, '--since', '30s'),
-        check=True,
-        stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        text=True,
-    )
-    logging.debug(f"Logs for {container_name}: {result.stdout}")
+    logs = glob.glob(f"{log_dir_for_container(container_name)}/*.log")
+
+    assert logs, f"No logs found for container {container_name}"
 
-    return result.stdout
+    with open(logs[0]) as f:
+        return f.read()
 
 
 def logs_contain_message(container_name, message):
@@ -126,21 +63,16 @@ def poll_logs_for_message(container_name, message, cycles=20):
     return False
 
 
-def test_end_to_end(setup, docker):
-    assert poll_logs_for_message("azurite-setup", "Blob containers created"), (
-        "Containers not ready")
-
-    logging.info("Containers are ready")
-
+def test_end_to_end(setup):
     assert upload_file_to_blob_storage(), "File upload unsuccessful"
 
     assert poll_logs_for_message("process-pilot-data", "Trigger Details:"), (
             "ProcessPilotData function not triggered by file upload")
 
-    assert logs_contain_message(
+    assert poll_logs_for_message(
             "process-pilot-data", "Executing 'Functions.ProcessPilotData'"), (
             "ProcessPilotData function not executed")
-    assert logs_contain_message(
+    assert poll_logs_for_message(
             "notify", "Executing 'Functions.Notify'"), (
             "Notify function not executed")
     assert poll_logs_for_message(