feat: add OpenTelemetry + X-Ray tracing

.dockerignore

@@ -0,0 +1,7 @@
+.git
+infrastructure/aws/cdk.out
+.venv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+

.github/actions/cdk-deploy/action.yml

@@ -36,15 +36,10 @@ runs:
       run: npm install -g aws-cdk@2
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v3
+      uses: astral-sh/setup-uv@v7
       with:
         version: "0.5.*" 
 
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
     - name: Install dependencies
       shell: bash
       working-directory: ${{ inputs.dir }}

Dockerfile

@@ -26,5 +26,5 @@ RUN if [ -z "$EARTHDATA_USERNAME" ] || [ -z "$EARTHDATA_PASSWORD" ]; then \
 # http://www.uvicorn.org/settings/
 ENV HOST 0.0.0.0
 ENV PORT 80
-CMD uv run uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload
+CMD uv run --no-dev uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload
 

README.md

@@ -90,7 +90,7 @@ The application will be available at this address: [http://localhost:8081/api.ht
 To run the application directly in your local environment, configure the application to access data over `HTTP` then run it using `uvicorn`:
 
 ```bash
-TITILER_CMR_S3_AUTH_ACCESS=external uv run uvicorn titiler.cmr.main:app --reload
+TITILER_CMR_S3_AUTH_ACCESS=external uv run uvicorn titiler.cmr.main:app --reload --log-level info
 ```
 
 The application will be available at this address: [http://localhost:8000/api.html](http://localhost:8000/api.html)

docs/examples/rasterio_backend_example.ipynb

@@ -41,10 +41,12 @@
    "outputs": [],
    "source": [
     "# titiler_endpoint = \"http://localhost:8081\"  # docker network endpoint\n",
+    "# titiler_endpoint = (\n",
+    "#     \"https://staging.openveda.cloud/api/titiler-cmr\"  # VEDA staging endpoint\n",
+    "# )\n",
     "titiler_endpoint = (\n",
-    "    \"https://staging.openveda.cloud/api/titiler-cmr\"  # VEDA staging endpoint\n",
-    ")\n",
-    "# titiler_endpoint = \"https://v4jec6i5c0.execute-api.us-west-2.amazonaws.com\"  # dev endpoint"
+    "    \"https://v4jec6i5c0.execute-api.us-west-2.amazonaws.com\"  # dev endpoint\n",
+    ")"
    ]
   },
   {
@@ -197,6 +199,7 @@
     "        (\"minzoom\", 8),\n",
     "        (\"maxzoom\", 13),\n",
     "    ),\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(r)"
@@ -260,6 +263,7 @@
     "        (\"minzoom\", 8),\n",
     "        (\"maxzoom\", 13),\n",
     "    ),\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "m = Map(location=(47.9221313337365, -91.65432884883238), zoom_start=r[\"maxzoom\"] - 1)\n",
@@ -337,7 +341,7 @@
     "        (\"bands\", \"B04\"),\n",
     "    ),\n",
     "    json=geojson,\n",
-    "    timeout=30,\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(r, indent=2))"

docs/examples/time_series_example.ipynb

@@ -178,6 +178,7 @@
     "        \"step\": \"P1W\",\n",
     "        \"temporal_mode\": \"point\",\n",
     "    },\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(response, indent=2))"
@@ -210,6 +211,7 @@
     "        \"step\": \"P1W\",\n",
     "        \"temporal_mode\": \"interval\",\n",
     "    },\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(response, indent=2))"
@@ -239,6 +241,7 @@
     "            [\"2024-10-01T00:00:01Z\", \"2024-10-07T00:00:01Z/2024-10-09T23:59:59Z\"]\n",
     "        ),\n",
     "    },\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(response, indent=2))"

docs/examples/xarray_backend_example.ipynb

@@ -184,6 +184,7 @@
     "        (\"rescale\", \"0,1\"),\n",
     "        (\"colormap_name\", \"blues_r\"),\n",
     "    ),\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(r)"
@@ -258,7 +259,7 @@
     "        (\"variable\", variable),\n",
     "    ),\n",
     "    json=geojson_dict,\n",
-    "    timeout=60,\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(r, indent=2))"
@@ -319,7 +320,7 @@
     "        (\"sel_method\", \"nearest\"),\n",
     "    ),\n",
     "    json=geojson_dict,\n",
-    "    timeout=60,\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(r, indent=2))"
@@ -354,7 +355,7 @@
     "        (\"sel_method\", \"nearest\"),\n",
     "    ),\n",
     "    json=geojson_dict,\n",
-    "    timeout=60,\n",
+    "    timeout=None,\n",
     ").json()\n",
     "\n",
     "print(json.dumps(r, indent=2))"

infrastructure/aws/cdk/app.py

@@ -12,6 +12,7 @@
 from aws_cdk import aws_sns as sns
 from aws_cdk import aws_sns_subscriptions as subscriptions
 from aws_cdk.aws_apigatewayv2_integrations import HttpLambdaIntegration
+from aws_cdk.aws_ecr_assets import Platform
 from config import AppSettings, StackSettings
 from constructs import Construct
 
@@ -65,27 +66,32 @@ def __init__(
             **DEFAULT_ENV,
             "TITILER_CMR_ROOT_PATH": app_settings.root_path,
             "TITILER_CMR_S3_AUTH_STRATEGY": app_settings.s3_auth_strategy,
+            "TITILER_CMR_TELEMETRY_ENABLED": "TRUE",
+            "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "aws-lambda,requests,urllib3,aiohttp-client",  # Disable aws-lambda auto-instrumentation (handled by otel_wrapper.py)
+            "OTEL_PROPAGATORS": "tracecontext,baggage,xray",
+            "OPENTELEMETRY_COLLECTOR_CONFIG_URI": "/opt/collector-config/config.yaml",
+            # AWS_LAMBDA_LOG_FORMAT not set - using custom JSON formatter in handler.py
+            "AWS_LAMBDA_EXEC_WRAPPER": "/opt/otel-instrument",  # Enable OTEL wrapper to avoid circular import
         }
 
         if app_settings.aws_request_payer:
             lambda_env["AWS_REQUEST_PAYER"] = app_settings.aws_request_payer
 
-        lambda_function = aws_lambda.Function(
+        lambda_function = aws_lambda.DockerImageFunction(
             self,
             f"{id}-lambda",
-            runtime=runtime,
-            code=aws_lambda.Code.from_docker_build(
-                path=os.path.abspath(context_dir),
+            code=aws_lambda.DockerImageCode.from_image_asset(
+                directory=os.path.abspath(context_dir),
                 file="infrastructure/aws/lambda/Dockerfile",
-                platform="linux/amd64",
+                platform=Platform.LINUX_AMD64,
             ),
-            handler="handler.handler",
             memory_size=memory,
             reserved_concurrent_executions=concurrent,
             timeout=Duration.seconds(timeout),
             environment=lambda_env,
             log_retention=logs.RetentionDays.ONE_WEEK,
             role=iam_reader_role,
+            tracing=aws_lambda.Tracing.ACTIVE,
         )
 
         for perm in permissions:

infrastructure/aws/lambda/Dockerfile

@@ -1,36 +1,80 @@
 ARG PYTHON_VERSION=3.12
 
-FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+# Stage 1: OTEL
+# Download the OpenTelemetry layer
+# ref: https://github.com/athewsey/opentelemetry-lambda-container/blob/98069d5eb6d812ccd28d5c80e2f9d6c8a8c76fb9/python-example/lambda-function/Dockerfile
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} as otel-builder
+RUN <<EOF
+dnf install -y unzip wget
+wget https://github.com/aws-observability/aws-otel-python-instrumentation/releases/download/v0.12.1/layer.zip -O /tmp/layer.zip
+mkdir -p /opt-builder
+unzip /tmp/layer.zip -d /opt-builder/
+EOF
+
+# Stage 2: titiler-cmr application and dependencies
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-WORKDIR /tmp
+RUN dnf install -y gcc-c++ && dnf clean all
 
-RUN dnf install -y gcc-c++  
+WORKDIR /build
 
-COPY uv.lock .python-version pyproject.toml LICENSE README.md ./
+COPY README.md uv.lock .python-version pyproject.toml ./
 COPY titiler/ ./titiler/
 
-RUN uv export --locked --no-editable --no-dev --extra lambda --extra telemetry --format requirements.txt -o requirements.txt && \
-  uv pip install --compile-bytecode --no-binary pydantic --target /asset -r requirements.txt 
+RUN <<EOF
+uv export --locked --no-editable --no-dev --extra lambda --format requirements.txt -o requirements.txt
+uv pip install \
+  --compile-bytecode \
+  --no-binary pydantic \
+  --target /deps \
+  --no-cache-dir \
+  --disable-pip-version-check \
+  -r requirements.txt
+EOF
+
+# Aggressive cleanup to minimize size and optimize for Lambda container
+# Clean up app dependencies in /deps
+WORKDIR /deps
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN <<EOF
+# Convert .pyc files and remove source .py files for faster cold starts
+find . -type f -name '*.pyc' | while read -r f; do n="$(echo "$f" | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//')"; cp "$f" "$n"; done
+find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+# Remove unnecessary files for Lambda runtime
+find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+find . -type d -a -name 'test' -print0 | xargs -0 rm -rf
+rm -rf numpy/doc/ bin/ geos_license Misc/
+# Remove unnecessary locale and documentation files
+find . -name '*.mo' -delete
+find . -name '*.po' -delete
+find . -name 'LICENSE*' -delete
+find . -name 'README*' -delete
+find . -name '*.md' -delete
+# Strip debug symbols from shared libraries (preserve numpy.libs)
+find . -type f -name '*.so*' -not -path "*/numpy.libs/*" -exec strip --strip-unneeded {} \; 2>/dev/null || true
+EOF
 
-# copy libexpat.so.1 into /asset which is included in LD_LIBRARY_PATH
-RUN cp /usr/lib64/libexpat.so.1 /asset/
+# Stage 3: Final runtime stage - minimal Lambda image optimized for container runtime
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
 
-# Reduce package size and remove useless files
-RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
-RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
-RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
-RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
-RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc
-RUN rm -rdf /asset/boto3*
-RUN rm -rdf /asset/botocore*
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1
 
-# Strip debug symbols from compiled C/C++ code (except for numpy.libs!)
-RUN cd /asset && \
-  find . -type f -name '*.so*' \
-  -not -path "./numpy.libs/*" \
-  -exec strip --strip-unneeded {} \;
+COPY --from=otel-builder /opt-builder/ /opt/
+COPY infrastructure/aws/lambda/collector-config.yaml /opt/collector-config/config.yaml
+COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/
+COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/
+COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/
 
-COPY infrastructure/aws/lambda/handler.py /asset/handler.py
+RUN <<EOF
+chmod 644 "${LAMBDA_RUNTIME_DIR}"/handler.py
+chmod -R 755 /opt/
+# Pre-compile the handler for faster cold starts
+python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler.py', doraise=True)"
+# Create cache directories with proper permissions
+mkdir -p /tmp/.cache && chmod 777 /tmp/.cache
+EOF
 
-CMD ["echo", "hello world"]
+CMD ["handler.lambda_handler"]

infrastructure/aws/lambda/collector-config.yaml

@@ -0,0 +1,38 @@
+extensions:
+  # AWS Proxy extension - forwards X-Ray segments to local X-Ray daemon via UDP
+  # This avoids the awsxray exporter making HTTPS API calls
+  awsproxy:
+    endpoint: 127.0.0.1:2000
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+      http:
+        endpoint: localhost:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 50
+
+exporters:
+  # Export to AWS X-Ray via local X-Ray daemon (UDP, no internet required)
+  # The awsproxy extension bridges the collector to the daemon at 127.0.0.1:2000
+  awsxray:
+    endpoint: http://127.0.0.1:2000
+    local_mode: true  # Use local X-Ray daemon instead of direct API calls
+    index_all_attributes: true
+
+  # Debug exporter to see traces in CloudWatch logs
+  debug:
+    verbosity: detailed
+
+service:
+  extensions: [awsproxy]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, awsxray]