just use aws-observability/aws-otel-python-instrumentation

Author: hrodmn

infrastructure/aws/cdk/app.py

@@ -43,7 +43,6 @@ def __init__(
         id: str,
         memory: int = 1024,
         timeout: int = 30,
-        runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_12,
         concurrent: Optional[int] = None,
         permissions: Optional[List[iam.PolicyStatement]] = None,
         environment: Optional[Dict] = None,
@@ -138,7 +137,12 @@ def __init__(
                 **environment,
                 "TITILER_MULTIDIM_ROOT_PATH": app_settings.root_path,
                 "TITILER_MULTIDIM_CACHE_HOST": redis_cluster.attr_redis_endpoint_address,
-                "OTEL_TRACES_ENABLED": "true",
+                "OTEL_METRICS_EXPORTER": "none",  # Disable metrics - only using traces
+                "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "aws-lambda,requests,urllib3,aiohttp-client",  # Disable aws-lambda auto-instrumentation (handled by otel_wrapper.py)
+                "OTEL_PROPAGATORS": "tracecontext,baggage,xray",
+                "OPENTELEMETRY_COLLECTOR_CONFIG_URI": "/opt/collector-config/config.yaml",
+                "AWS_LAMBDA_LOG_FORMAT": "JSON",
+                "AWS_LAMBDA_EXEC_WRAPPER": "/opt/otel-instrument",  # Enable OTEL wrapper to avoid circular import
             },
             log_retention=logs.RetentionDays.ONE_WEEK,
             vpc=vpc,
@@ -209,6 +213,17 @@ def __init__(
         )
     )
 
+# Add X-Ray permissions for tracing
+perms.append(
+    iam.PolicyStatement(
+        actions=[
+            "xray:PutTraceSegments",
+            "xray:PutTelemetryRecords",
+        ],
+        resources=["*"],
+    )
+)
+
 
 lambda_stack = LambdaStack(
     app,

infrastructure/aws/lambda/Dockerfile

@@ -1,22 +1,36 @@
 ARG PYTHON_VERSION=3.12
 
-# Build stage - includes all build tools and dependencies
-FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
+# Stage 1: Build dependencies
+
+# Download the OpenTelemetry Layer with AppSignals Support:
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} as otel-builder
+RUN dnf install -y unzip wget && \
+  wget https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest/download/layer.zip -O /tmp/layer.zip && \
+  mkdir -p /opt-builder && \
+  unzip /tmp/layer.zip -d /opt-builder/
 
-# Copy uv for faster dependency management
+# install python dependencies
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-# Install system dependencies needed for compilation
 RUN dnf install -y gcc-c++ && dnf clean all
 
-# Set working directory for build
 WORKDIR /build
 
-# Copy dependency files first for better caching
 COPY README.md uv.lock .python-version pyproject.toml ./
 COPY src/titiler/ ./src/titiler/
 
+COPY infrastructure/aws/lambda/otel-requirements.txt ./
+
 # Install dependencies to temporary directory with Lambda-specific optimizations
+# Install OTEL packages to /opt/python (where otel-instrument wrapper expects them)
+RUN uv pip install \
+  --target /opt-python \
+  --no-cache-dir \
+  --disable-pip-version-check \
+  -r otel-requirements.txt
+
+# Install app packages (excluding OTEL packages since they're already installed)
 RUN uv export --locked --no-editable --no-dev --extra lambda --format requirements.txt -o requirements.txt && \
   uv pip install \
   --compile-bytecode \
@@ -26,7 +40,9 @@ RUN uv export --locked --no-editable --no-dev --extra lambda --format requiremen
   --disable-pip-version-check \
   -r requirements.txt
 
+
 # Aggressive cleanup to minimize size and optimize for Lambda container
+# Clean up app dependencies in /deps
 WORKDIR /deps
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN <<EOF
@@ -50,30 +66,53 @@ find . -type f -name '*.so*' -not -path "*/numpy.libs/*" -exec strip --strip-unn
 du -sh . > /tmp/package_size.txt
 EOF
 
-# Final runtime stage - minimal Lambda image optimized for container runtime
+# Clean up OTEL dependencies in /opt-python (lighter cleanup to preserve CLI tools)
+WORKDIR /opt-python
+RUN <<EOF
+# Remove test files and docs
+find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+find . -type d -a -name 'test' -print0 | xargs -0 rm -rf
+# Keep bin/ directory for opentelemetry-instrument CLI
+find . -name '*.mo' -delete
+find . -name '*.po' -delete
+find . -name 'LICENSE*' -delete
+find . -name 'README*' -delete
+find . -name '*.md' -delete
+EOF
+
+# Stage 3: Final runtime stage - minimal Lambda image optimized for container runtime
 FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
 
 # Set Lambda-specific environment variables for optimal performance
-ENV PYTHONPATH=${LAMBDA_RUNTIME_DIR} \
-  PYTHONUNBUFFERED=1 \
-  PYTHONDONTWRITEBYTECODE=1 \
-  AWS_LWA_ENABLE_COMPRESSION=true
+# PYTHONPATH includes both /opt/python (OTEL packages) and LAMBDA_RUNTIME_DIR (app packages)
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1
+
+# Copy OTEL code
+COPY --from=otel-builder /opt-builder/ /opt/
+COPY infrastructure/aws/lambda/collector-config.yaml /opt/collector-config/config.yaml
+
+# Copy dependencies from builder stage
+# OTEL packages to /opt/python (includes opentelemetry-instrument CLI)
+COPY --from=builder /opt-python /opt/python
+
+# App packages to LAMBDA_RUNTIME_DIR
+COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/
 
-# Copy only the cleaned dependencies from builder stage
 # Copy required system library
-COPY --from=builder /deps /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/
+COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/
 
 # Copy application handler and OpenTelemetry configuration
-COPY infrastructure/aws/lambda/handler.py infrastructure/aws/lambda/otel_config.py ${LAMBDA_RUNTIME_DIR}/
+COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/
 
 # Ensure handler is executable and optimize permissions
 RUN <<EOF
 chmod 644 "${LAMBDA_RUNTIME_DIR}"/handler.py
+chmod -R 755 /opt/
 # Pre-compile the handler for faster cold starts
 python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler.py', doraise=True)"
 # Create cache directories with proper permissions
 mkdir -p /tmp/.cache && chmod 777 /tmp/.cache
 EOF
 
-# Set the Lambda handler
 CMD ["handler.lambda_handler"]

infrastructure/aws/lambda/collector-config.yaml

@@ -0,0 +1,38 @@
+extensions:
+  # AWS Proxy extension - forwards X-Ray segments to local X-Ray daemon via UDP
+  # This avoids the awsxray exporter making HTTPS API calls
+  awsproxy:
+    endpoint: 127.0.0.1:2000
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+      http:
+        endpoint: localhost:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 50
+
+exporters:
+  # Export to AWS X-Ray via local X-Ray daemon (UDP, no internet required)
+  # The awsproxy extension bridges the collector to the daemon at 127.0.0.1:2000
+  awsxray:
+    endpoint: http://127.0.0.1:2000
+    local_mode: true  # Use local X-Ray daemon instead of direct API calls
+    index_all_attributes: true
+
+  # Debug exporter to see traces in CloudWatch logs
+  debug:
+    verbosity: detailed
+
+service:
+  extensions: [awsproxy]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, awsxray]

infrastructure/aws/lambda/handler.py

@@ -1,22 +1,20 @@
-"""AWS Lambda handler optimized for container runtime."""
+"""AWS Lambda handler optimized for container runtime with OTEL instrumentation."""
 
 import logging
 import warnings
 from typing import Any, Dict
 
-# Initialize OpenTelemetry BEFORE importing the FastAPI app
-from otel_config import setup_otel
+from mangum import Mangum
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
 
-setup_otel()
-
-from mangum import Mangum  # noqa: E402
-
-from titiler.multidim.main import app  # noqa: E402
+from titiler.multidim.main import app
 
 # Configure root logger to WARN level by default
+# Use simple format - AWS Lambda will handle JSON formatting when AWS_LAMBDA_LOG_FORMAT=JSON
 logging.basicConfig(
     level=logging.WARN,
-    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    format="[%(levelname)s] %(name)s: %(message)s",
 )
 
 # Set titiler loggers to INFO level
@@ -31,15 +29,8 @@
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 
-
-# Pre-import commonly used modules for faster cold starts
-try:
-    import numpy  # noqa: F401
-    import pandas  # noqa: F401
-    import rioxarray  # noqa: F401
-    import xarray  # noqa: F401
-except ImportError:
-    pass
+LoggingInstrumentor().instrument(set_logging_format=False)
+FastAPIInstrumentor.instrument_app(app)
 
 handler = Mangum(
     app,
@@ -55,10 +46,5 @@
 
 
 def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
-    """Lambda handler with container-specific optimizations."""
-    response = handler(event, context)
-
-    return response
-
-
-handler.lambda_handler = lambda_handler
+    """Lambda handler with container-specific optimizations and OTEL tracing."""
+    return handler(event, context)

infrastructure/aws/lambda/otel-requirements.txt

@@ -0,0 +1,4 @@
+aws-opentelemetry-distro>=0.12.1
+opentelemetry-instrumentation-botocore
+opentelemetry-instrumentation-fastapi
+opentelemetry-instrumentation-logging