build: speed up raster and stac lambda builds

app.py

@@ -120,20 +120,20 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
 )
 
 ingest_api = ingest_api_construct(
-    veda_stack,
-    "ingest-api",
-    config=ingestor_config,
-    db_secret=database.pgstac.secret,
-    db_vpc=vpc.vpc,
+   veda_stack,
+   "ingest-api",
+   config=ingestor_config,
+   db_secret=database.pgstac.secret,
+   db_vpc=vpc.vpc,
 )
 
 ingestor = ingestor_construct(
-    veda_stack,
-    "IngestorConstruct",
-    config=ingestor_config,
-    table=ingest_api.table,
-    db_secret=database.pgstac.secret,
-    db_vpc=vpc.vpc,
+   veda_stack,
+   "IngestorConstruct",
+   config=ingestor_config,
+   table=ingest_api.table,
+   db_secret=database.pgstac.secret,
+   db_vpc=vpc.vpc,
 )
 
 for key, value in {

database/infrastructure/construct.py

@@ -45,18 +45,18 @@ def __init__(
         veda_schema_version = veda_db_settings.schema_version
 
         handler = aws_lambda.Function(
-            self,
-            "lambda",
-            handler="handler.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_12,
-            code=aws_lambda.Code.from_docker_build(
-                path=os.path.abspath("./"),
-                file="database/runtime/Dockerfile",
-                build_args={"PGSTAC_VERSION": pgstac_version},
-            ),
-            timeout=Duration.minutes(5),
-            vpc=database.vpc,
-            log_retention=aws_logs.RetentionDays.ONE_WEEK,
+           self,
+           "lambda",
+           handler="handler.handler",
+           runtime=aws_lambda.Runtime.PYTHON_3_12,
+           code=aws_lambda.Code.from_docker_build(
+               path=os.path.abspath("./"),
+               file="database/runtime/Dockerfile",
+               build_args={"PGSTAC_VERSION": pgstac_version},
+           ),
+           timeout=Duration.minutes(5),
+           vpc=database.vpc,
+           log_retention=aws_logs.RetentionDays.ONE_WEEK,
         )
 
         self.secret = aws_secretsmanager.Secret(
@@ -90,18 +90,18 @@ def __init__(
         self.connections = database.connections
 
         CustomResource(
-            scope=scope,
-            id="bootstrapper",
-            service_token=handler.function_arn,
-            properties={
-                # By setting pgstac_version in the properties assures
-                # that Create/Update events will be passed to the service token
-                "pgstac_version": pgstac_version,
-                "conn_secret_arn": database.secret.secret_arn,
-                "new_user_secret_arn": self.secret.secret_arn,
-                "veda_schema_version": veda_schema_version,
-            },
-            removal_policy=RemovalPolicy.RETAIN,  # This retains the custom resource (which doesn't really exist), not the database
+           scope=scope,
+           id="bootstrapper",
+           service_token=handler.function_arn,
+           properties={
+               # By setting pgstac_version in the properties assures
+               # that Create/Update events will be passed to the service token
+               "pgstac_version": pgstac_version,
+               "conn_secret_arn": database.secret.secret_arn,
+               "new_user_secret_arn": self.secret.secret_arn,
+               "veda_schema_version": veda_schema_version,
+           },
+           removal_policy=RemovalPolicy.RETAIN,  # This retains the custom resource (which doesn't really exist), not the database
         )
 
 

raster_api/infrastructure/construct.py

@@ -13,6 +13,7 @@
     aws_lambda,
     aws_logs,
 )
+from aws_cdk.aws_ecr_assets import Platform
 from constructs import Construct
 
 from .config import veda_raster_settings
@@ -37,17 +38,15 @@ def __init__(
         # TODO config
         stack_name = Stack.of(self).stack_name
 
-        veda_raster_function = aws_lambda.Function(
+        veda_raster_function = aws_lambda.DockerImageFunction(
             self,
             "lambda",
-            runtime=aws_lambda.Runtime.PYTHON_3_12,
-            code=aws_lambda.Code.from_docker_build(
-                path=os.path.abspath(code_dir),
+            code=aws_lambda.DockerImageCode.from_image_asset(
+                directory=os.path.abspath(code_dir),
                 file="raster_api/runtime/Dockerfile",
-                platform="linux/amd64",
+                platform=Platform.LINUX_AMD64,
             ),
             vpc=vpc,
-            handler="handler.handler",
             memory_size=veda_raster_settings.memory,
             timeout=Duration.seconds(veda_raster_settings.timeout),
             log_retention=aws_logs.RetentionDays.ONE_WEEK,

raster_api/runtime/Dockerfile

@@ -1,25 +1,71 @@
-FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.12:latest
+ARG PYTHON_VERSION=3.12
 
-RUN dnf install -y gcc-c++
+# Stage 1: application and dependencies
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+RUN dnf install -y gcc-c++ && dnf clean all
 
 WORKDIR /tmp
 
 COPY raster_api/runtime /tmp/raster
-RUN pip install mangum /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic
+
+# we could have a `uv export` here, if we converted the project format
+RUN uv pip install mangum /tmp/raster["psycopg-binary"] --target /deps --no-binary pydantic
+
+RUN <<EOF
+uv pip install \
+  --compile-bytecode \
+  --no-binary pydantic \
+  --target /deps \
+  --no-cache-dir \
+  --disable-pip-version-check \
+  /tmp/raster["psycopg-binary"] \
+  mangum
+EOF
 RUN rm -rf /tmp/raster
-RUN cp /usr/lib64/libexpat.so.1 /asset/
 
-# # Reduce package size and remove useless files
-RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
-RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
-RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
-RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
-RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc
+# Aggressive cleanup to minimize size and optimize for Lambda container
+# Clean up app dependencies in /deps
+WORKDIR /deps
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN <<EOF
+# Convert .pyc files and remove source .py files for faster cold starts
+find . -type f -name '*.pyc' | while read -r f; do n="$(echo "$f" | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//')"; cp "$f" "$n"; done
+find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+# Remove unnecessary files for Lambda runtime
+find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+find . -type d -a -name 'test' -print0 | xargs -0 rm -rf
+rm -rf numpy/doc/ bin/ geos_license Misc/
+# Remove unnecessary locale and documentation files
+find . -name '*.mo' -delete
+find . -name '*.po' -delete
+find . -name 'LICENSE*' -delete
+find . -name 'README*' -delete
+find . -name '*.md' -delete
+# Strip debug symbols from shared libraries (preserve numpy.libs)
+find . -type f -name '*.so*' -not -path "*/numpy.libs/*" -exec strip --strip-unneeded {} \; 2>/dev/null || true
+EOF
+
+# Stage 2: Final runtime stage - minimal Lambda image optimized for container runtime
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
+
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1
+
+COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/
+COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/
+COPY raster_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/
 
-COPY raster_api/runtime/handler.py /asset/handler.py
-RUN dnf remove -y gcc-c++
 
-WORKDIR /asset
-RUN python -c "from handler import handler; print('All Good')"
+RUN <<EOF
+chmod 644 "${LAMBDA_RUNTIME_DIR}"/handler.py
+chmod -R 755 /opt/
+# Pre-compile the handler for faster cold starts
+python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler.py', doraise=True)"
+# Create cache directories with proper permissions
+mkdir -p /tmp/.cache && chmod 777 /tmp/.cache
+EOF
 
-CMD ["echo", "hello world"]
+CMD ["handler.lambda_handler"]

raster_api/runtime/handler.py

@@ -3,6 +3,7 @@
 import asyncio
 import logging
 import os
+from typing import Any, Dict
 
 from mangum import Mangum
 from src.app import app
@@ -36,3 +37,7 @@ async def startup_event() -> None:
 handler = logger.inject_lambda_context(handler, clear_state=True)
 # Add metrics last to properly flush metrics.
 handler = metrics.log_metrics(handler, capture_cold_start_metric=True)
+
+def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
+    """Lambda handler with container-specific optimizations and OTEL tracing."""
+    return handler(event, context)

stac_api/infrastructure/construct.py

@@ -12,6 +12,7 @@
     aws_lambda,
     aws_logs,
 )
+from aws_cdk.aws_ecr_assets import Platform
 from constructs import Construct
 
 from .config import veda_stac_settings
@@ -67,14 +68,13 @@ def __init__(
                 veda_stac_settings.openid_configuration_url
             )
 
-        lambda_function = aws_lambda.Function(
+        lambda_function = aws_lambda.DockerImageFunction(
             self,
             "lambda",
-            handler="handler.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_12,
-            code=aws_lambda.Code.from_docker_build(
-                path=os.path.abspath(code_dir),
+            code=aws_lambda.DockerImageCode.from_image_asset(
+                directory=os.path.abspath(code_dir),
                 file="stac_api/runtime/Dockerfile",
+                platform=Platform.LINUX_AMD64,
             ),
             vpc=vpc,
             memory_size=veda_stac_settings.memory,

stac_api/runtime/Dockerfile

@@ -1,18 +1,58 @@
-FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.12:latest
+ARG PYTHON_VERSION=3.12
+
+# Stage 1: application and dependencies
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+RUN dnf install -y gcc-c++ && dnf clean all
 
 WORKDIR /tmp
 
 COPY stac_api/runtime /tmp/stac
 
-RUN pip install "mangum" "plpygis>=0.2.1" /tmp/stac -t /asset --no-binary pydantic
+# we could have a `uv export` here, if we converted the project format
+RUN uv pip install mangum "plpygis>=0.2.1" /tmp/stac["psycopg-binary"] --target /deps --no-binary pydantic
 RUN rm -rf /tmp/stac
 
-# Reduce package size and remove useless files
-RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
-RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
-RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
-RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+WORKDIR /deps
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN <<EOF
+# Convert .pyc files and remove source .py files for faster cold starts
+find . -type f -name '*.pyc' | while read -r f; do n="$(echo "$f" | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//')"; cp "$f" "$n"; done
+find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+# Remove unnecessary files for Lambda runtime
+find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+find . -type d -a -name 'test' -print0 | xargs -0 rm -rf
+rm -rf numpy/doc/ bin/ geos_license Misc/
+# Remove unnecessary locale and documentation files
+find . -name '*.mo' -delete
+find . -name '*.po' -delete
+find . -name 'LICENSE*' -delete
+find . -name 'README*' -delete
+find . -name '*.md' -delete
+# Strip debug symbols from shared libraries (preserve numpy.libs)
+find . -type f -name '*.so*' -not -path "*/numpy.libs/*" -exec strip --strip-unneeded {} \; 2>/dev/null || true
+EOF
+
+# Stage 2: Final runtime stage - minimal Lambda image optimized for container runtime
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
+
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1
+
+COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/
+COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/
+COPY stac_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/
+
 
-COPY stac_api/runtime/handler.py /asset/handler.py
+RUN <<EOF
+chmod 644 "${LAMBDA_RUNTIME_DIR}"/handler.py
+chmod -R 755 /opt/
+# Pre-compile the handler for faster cold starts
+python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler.py', doraise=True)"
+# Create cache directories with proper permissions
+mkdir -p /tmp/.cache && chmod 777 /tmp/.cache
+EOF
 
-CMD ["echo", "hello world"]
+CMD ["handler.lambda_handler"]

stac_api/runtime/handler.py

@@ -5,6 +5,7 @@
 from mangum import Mangum
 from src.app import app
 from src.monitoring import logger, metrics, tracer
+from typing import Any, Dict
 
 logging.getLogger("mangum.lifespan").setLevel(logging.ERROR)
 logging.getLogger("mangum.http").setLevel(logging.ERROR)
@@ -18,3 +19,7 @@
 handler = logger.inject_lambda_context(handler, clear_state=True)
 # Add metrics last to properly flush metrics.
 handler = metrics.log_metrics(handler, capture_cold_start_metric=True)
+
+def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
+    """Lambda handler with container-specific optimizations and OTEL tracing."""
+    return handler(event, context)