From 3036d5264af4a23b8d76f36793d72a986e7bf9af Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Wed, 15 Jul 2026 01:14:26 +0200 Subject: [PATCH] feat: added monitoring. --- .github/workflows/deploy.yaml | 14 +- .github/workflows/tests/test_raster.py | 2 +- README.md | 33 + docker-compose.metrics.yml | 95 ++ docker-compose.yml | 10 +- dockerfiles/Dockerfile.browser | 10 +- dockerfiles/landing/img/grafana_logo.svg | 1 + dockerfiles/landing/index.html | 15 +- .../grafana/dashboards/eoapi-overview.json | 827 +++++++++++++++++ .../grafana/dashboards/eoapi-raster.json | 652 +++++++++++++ .../dashboards/eoapi-stac-auth-proxy.json | 652 +++++++++++++ .../grafana/dashboards/eoapi-stac.json | 652 +++++++++++++ .../grafana/dashboards/eoapi-vector.json | 862 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/prometheus.yml | 10 + dockerfiles/metrics/prometheus.yml | 43 + infrastructure/docker-compose.hetzner.yml | 28 +- runtimes/eoapi/raster/eoapi/raster/app.py | 3 + runtimes/eoapi/raster/eoapi/raster/config.py | 10 +- .../eoapi/raster/templates/stac-viewer.html | 13 +- runtimes/eoapi/raster/pyproject.toml | 4 +- runtimes/eoapi/stac/eoapi/stac/app.py | 9 + runtimes/eoapi/stac/pyproject.toml | 8 +- runtimes/eoapi/vector/eoapi/vector/app.py | 3 + runtimes/eoapi/vector/pyproject.toml | 4 +- scripts/demo_db_refresh.sh | 38 + scripts/install_demo_cron.sh | 37 + scripts/simulate_traffic.py | 701 ++++++++++++++ scripts/simulate_traffic_loop.py | 130 +++ 29 files changed, 4842 insertions(+), 35 deletions(-) create mode 100644 docker-compose.metrics.yml create mode 100644 dockerfiles/landing/img/grafana_logo.svg create mode 100644 dockerfiles/metrics/grafana/dashboards/eoapi-overview.json create mode 100644 dockerfiles/metrics/grafana/dashboards/eoapi-raster.json create mode 100644 dockerfiles/metrics/grafana/dashboards/eoapi-stac-auth-proxy.json create mode 100644 dockerfiles/metrics/grafana/dashboards/eoapi-stac.json create mode 100644 dockerfiles/metrics/grafana/dashboards/eoapi-vector.json create mode 100644 dockerfiles/metrics/grafana/provisioning/dashboards/dashboards.yml create mode 100644 dockerfiles/metrics/grafana/provisioning/datasources/prometheus.yml create mode 100644 dockerfiles/metrics/prometheus.yml create mode 100644 scripts/demo_db_refresh.sh create mode 100644 scripts/install_demo_cron.sh create mode 100755 scripts/simulate_traffic.py create mode 100644 scripts/simulate_traffic_loop.py diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index a10090f..6b7d09e 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -33,7 +33,7 @@ jobs: environment: hetzner env: ACME_EMAIL: ${{ vars.ACME_EMAIL }} - COMPOSE_FILE: docker-compose.yml:docker-compose.mock-oidc.yml:infrastructure/docker-compose.hetzner.yml + COMPOSE_FILE: docker-compose.yml:docker-compose.mock-oidc.yml:docker-compose.metrics.yml:infrastructure/docker-compose.hetzner.yml DEPLOY_PATH: /home/user/deploy DEPLOY_USER: user EOAPI_DOMAIN: ${{ vars.EOAPI_DOMAIN }} @@ -146,6 +146,7 @@ jobs: "cd '${DEPLOY_PATH}' && \ export COMPOSE_FILE='${COMPOSE_FILE}' && \ export EOAPI_DOMAIN='${EOAPI_DOMAIN}' && \ + export EOAPI_GRAFANA_ROOT_URL='https://${EOAPI_DOMAIN}/monitoring/' && \ export ACME_EMAIL='${ACME_EMAIL}' && \ docker compose down && \ bash .github/workflows/ingest.sh --wipe-pgdata-only && \ @@ -155,6 +156,17 @@ jobs: docker compose restart vector && \ docker compose ps" + - name: Install demo refresh cron + run: | + ssh $SSH_OPTS "${DEPLOY_USER}@${EOAPI_DOMAIN}" \ + "cd '${DEPLOY_PATH}' && \ + export DEPLOY_PATH='${DEPLOY_PATH}' && \ + export EOAPI_DOMAIN='${EOAPI_DOMAIN}' && \ + export EOAPI_GRAFANA_ROOT_URL='https://${EOAPI_DOMAIN}/monitoring/' && \ + export ACME_EMAIL='${ACME_EMAIL}' && \ + export COMPOSE_FILE='${COMPOSE_FILE}' && \ + bash scripts/install_demo_cron.sh" + - name: Wait for services run: | wait_for_service() { diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index ae25afe..6e1ffe6 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -28,7 +28,7 @@ def test_raster_api(): def test_mosaic_api(): """test mosaic.""" - query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql-json"} + query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql2-json"} resp = httpx.post(f"{raster_endpoint}/searches/register", json=query) assert resp.headers["content-type"] == "application/json" assert resp.status_code == 200 diff --git a/README.md b/README.md index 158ecd9..8c0cb34 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,39 @@ Item map viewers are available at `/raster/collections/{collection_id}/items/{it If you've added a vector dataset to the `public` schema in the Postgres database, they will be available through the **Vector** service at [http://localhost:8080/vector](http://localhost:8080/vector). +### Monitoring + +Optional Prometheus + Grafana stack for service metrics. Start with the metrics overlay (combine with mock OIDC for Grafana login): + +``` +docker compose -f docker-compose.yml -f docker-compose.mock-oidc.yml -f docker-compose.metrics.yml up --watch +``` + +| Endpoint | Description | +|----------|-------------| +| [http://localhost:8080/monitoring](http://localhost:8080/monitoring) | Grafana dashboards (via Traefik); start with **eoAPI Overview** | +| [http://127.0.0.1:9090](http://127.0.0.1:9090) | Prometheus UI (debugging) | +| `stac:8081/_mgmt/metrics` | STAC API metrics (internal) | +| `raster:8082/metrics` | Raster service metrics (internal) | +| `vector:8083/metrics` | Vector service metrics (internal) | + +Provisioned dashboards live in the **eoAPI** folder: + +- **eoAPI Overview** — service health, cross-service comparison, and database saturation (connections, transactions, cache hit ratio) +- **eoAPI — STAC Auth Proxy**, **STAC**, **Raster**, **Vector** — per-service detail (errors, latency, traffic by operation) + +All use low-cardinality `operation` labels from the runtimes. STAC maps STAC API endpoints; raster and vector map coarse route groups (`tiles`, `register_search`, `list_items`, etc.). The Kubernetes infrastructure dashboard in [eoapi-k8s](https://github.com/developmentseed/eoapi-k8s) (`charts/eoapi/data/dashboards/eoAPI-Dashboard.json`) targets pod CPU/memory and nginx ingress metrics and is not included in this Docker Compose stack. + +**Grafana login:** When using `docker-compose.mock-oidc.yml`, click **Sign in with Mock OIDC** and enter any username on the mock login form. Grafana maps the username to `{username}@localhost` when the provider does not return an email claim. Username/password login is disabled. + +If Grafana is accessed on a different host or port, set `EOAPI_GRAFANA_ROOT_URL` (for example `https://eoapi.example.com/monitoring/`) so OAuth callbacks use the Traefik URL rather than Grafana's internal port. + +To populate Grafana with sample traffic, run the [ingest profile](#local-testing) first for richer STAC and raster traffic, then generate load with: + +``` +uv run scripts/simulate_traffic.py --duration 3m --users 6 +``` + ## Deployment ### Requirements diff --git a/docker-compose.metrics.yml b/docker-compose.metrics.yml new file mode 100644 index 0000000..f30d902 --- /dev/null +++ b/docker-compose.metrics.yml @@ -0,0 +1,95 @@ +# Optional overlay: Prometheus + Grafana monitoring for eoAPI runtimes. +# +# docker compose -f docker-compose.yml -f docker-compose.mock-oidc.yml -f docker-compose.metrics.yml up +# +# Grafana is served at http://localhost:8080/monitoring (via Traefik). +# Prometheus UI is at http://127.0.0.1:9090 for debugging. +# +# Grafana login uses the mock OIDC server when docker-compose.mock-oidc.yml is +# also applied — any username works. Username/password login is disabled. + +services: + traefik: + depends_on: + - grafana + + prometheus: + image: prom/prometheus:latest + volumes: + - ./dockerfiles/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:9090:9090" + depends_on: + - stac + - stac-auth-proxy + - raster + - vector + - postgres-exporter + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-lifecycle + + grafana: + image: grafana/grafana:latest + environment: + # Static creds are inert while GF_AUTH_DISABLE_LOGIN_FORM/GF_AUTH_BASIC_ENABLED + # below keep the basic-auth login form off; don't rely on this if that changes. + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_USERS_ALLOW_SIGN_UP: "true" + GF_USERS_AUTO_ASSIGN_ORG_ROLE: Editor + # Must match the Traefik URL, not Grafana's internal :3000 port. + GF_SERVER_ROOT_URL: "${EOAPI_GRAFANA_ROOT_URL:-http://localhost:8080/monitoring/}" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_AUTH_BASIC_ENABLED: "false" + GF_AUTH_GENERIC_OAUTH_ENABLED: "true" + GF_AUTH_GENERIC_OAUTH_NAME: Mock OIDC + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: grafana + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "" + GF_AUTH_GENERIC_OAUTH_SCOPES: openid profile + GF_AUTH_GENERIC_OAUTH_AUTH_URL: http://localhost:8085/authorize + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: http://mock-oidc:8888/token + GF_AUTH_GENERIC_OAUTH_USE_PKCE: "true" + GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: "true" + # mock-oidc only returns `sub` unless custom claims are provided. + GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH: sub + GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH: sub + GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH: "email || join('', [sub, '@localhost'])" + GF_AUTH_GENERIC_OAUTH_SKIP_ORG_ROLE_SYNC: "true" + GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "Editor" + volumes: + - ./dockerfiles/metrics/grafana/provisioning:/etc/grafana/provisioning:ro + - ./dockerfiles/metrics/grafana/dashboards:/etc/grafana/dashboards:ro + depends_on: + - prometheus + labels: + - traefik.enable=true + - traefik.http.routers.grafana.rule=PathPrefix(`/monitoring`) + - traefik.http.routers.grafana.entrypoints=web + - traefik.http.services.grafana.loadbalancer.server.port=3000 + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:latest + environment: + DATA_SOURCE_NAME: postgresql://username:password@database:5432/postgis?sslmode=disable + depends_on: + database: + condition: service_healthy + + stac: + environment: + - PROMETHEUS_DISABLE_CREATED_SERIES=True + + raster: + environment: + - PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus + - PROMETHEUS_DISABLE_CREATED_SERIES=True + command: bash -c "mkdir -p /tmp/prometheus && rm -rf /tmp/prometheus/* && bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn eoapi.raster.app:app --host 0.0.0.0 --port 8082 --workers 10" + + vector: + environment: + - PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus + - PROMETHEUS_DISABLE_CREATED_SERIES=True + command: bash -c "mkdir -p /tmp/prometheus && rm -rf /tmp/prometheus/* && bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn eoapi.vector.app:app --host 0.0.0.0 --port 8083 --workers 10" diff --git a/docker-compose.yml b/docker-compose.yml index b5e7e06..88a5aac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,11 +173,11 @@ services: environment: - PYTHONUNBUFFERED=1 # Application - - POSTGRES_USER=username - - POSTGRES_PASS=password - - POSTGRES_DBNAME=postgis - - POSTGRES_HOST=database - - POSTGRES_PORT=5432 + - PGUSER=username + - PGPASSWORD=password + - PGDATABASE=postgis + - PGHOST=database + - PGPORT=5432 - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=10 - CPL_TMPDIR=/tmp diff --git a/dockerfiles/Dockerfile.browser b/dockerfiles/Dockerfile.browser index bdcb867..9a8ea40 100644 --- a/dockerfiles/Dockerfile.browser +++ b/dockerfiles/Dockerfile.browser @@ -1,25 +1,27 @@ +# Requires stac-browser >=5.0.0, which lets SB_pathPrefix (and other config) +# be set at container startup via DYNAMIC_CONFIG (radiantearth/stac-browser#930). ARG STAC_BROWSER_REPO=https://github.com/radiantearth/stac-browser.git ARG STAC_BROWSER_REF=v5.0.0 FROM node:lts-alpine AS build-step +ARG DYNAMIC_CONFIG=true ARG historyMode="history" ARG SB_CONFIG="" ARG STAC_BROWSER_REPO ARG STAC_BROWSER_REF -ENV DYNAMIC_CONFIG=true +ENV DYNAMIC_CONFIG="${DYNAMIC_CONFIG}" ENV SB_historyMode="${historyMode}" ENV SB_CONFIG="${SB_CONFIG}" WORKDIR /app RUN apk add --no-cache git -RUN git clone "${STAC_BROWSER_REPO}" . && \ - git checkout "${STAC_BROWSER_REF}" +RUN git clone --branch "${STAC_BROWSER_REF}" --depth 1 "${STAC_BROWSER_REPO}" . RUN npm install # Uncomment the /runtime-config.js tags in index.html, which Vite (via # DYNAMIC_CONFIG) otherwise leaves commented out for builds with a fixed pathPrefix. -RUN sed -i 's///' index.html +RUN if [ "${DYNAMIC_CONFIG}" = "true" ]; then sed -i 's///' index.html; fi RUN npm run build diff --git a/dockerfiles/landing/img/grafana_logo.svg b/dockerfiles/landing/img/grafana_logo.svg new file mode 100644 index 0000000..c99695a --- /dev/null +++ b/dockerfiles/landing/img/grafana_logo.svg @@ -0,0 +1 @@ +Grafana \ No newline at end of file diff --git a/dockerfiles/landing/index.html b/dockerfiles/landing/index.html index 19e7e50..e82e1af 100644 --- a/dockerfiles/landing/index.html +++ b/dockerfiles/landing/index.html @@ -83,6 +83,19 @@
tipg — spatial objects
+
  • +
    + Grafana logo +
    +

    Monitoring

    +
    Grafana — service metrics
    +

    Prometheus-backed observability dashboards for eoAPI services.

    + +
    +
    +
    • @@ -121,7 +134,7 @@

    Learn more

    -

    Made by

    +

    Brought to you by

    Development Seed logo diff --git a/dockerfiles/metrics/grafana/dashboards/eoapi-overview.json b/dockerfiles/metrics/grafana/dashboards/eoapi-overview.json new file mode 100644 index 0000000..126192e --- /dev/null +++ b/dockerfiles/metrics/grafana/dashboards/eoapi-overview.json @@ -0,0 +1,827 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "icon": "dashboard", + "title": "STAC Auth Proxy", + "type": "link", + "url": "/d/eoapi-stac-auth-proxy/eoapi-stac-auth-proxy" + }, + { + "icon": "dashboard", + "title": "STAC", + "type": "link", + "url": "/d/eoapi-stac/eoapi-stac" + }, + { + "icon": "dashboard", + "title": "Raster", + "type": "link", + "url": "/d/eoapi-raster/eoapi-raster" + }, + { + "icon": "dashboard", + "title": "Vector", + "type": "link", + "url": "/d/eoapi-vector/eoapi-vector" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "Total", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "Total request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\", status=\"4xx\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"4xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "Total 4xx rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\", status=\"5xx\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"5xx\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"5xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "Total 5xx rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "max(histogram_quantile(0.95, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"stac-auth-proxy\", operation!=\"\"}[5m]) or rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m]))))", + "refId": "A" + } + ], + "title": "Worst p95 latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 101, + "title": "Compare services", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 30, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m]))", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 33, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "100 * (sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\", status=\"4xx\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"4xx\"}[5m])) or on(job) (0 * sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])))) / clamp_min(sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "4xx rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 31, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "100 * (sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\", status=\"5xx\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"5xx\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"5xx\"}[5m])) or on(job) (0 * sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])))) / clamp_min(sum by (job) (rate(http_requests_total{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "5xx rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 32, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"stac-auth-proxy|stac\", operation!=\"\"}[5m]) or rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m]) or rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m])))", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "p95 latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 103, + "title": "Database", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max_connections" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (state) (pg_stat_activity_count{datname=\"postgis\"})", + "legendFormat": "{{state}}", + "refId": "A" + }, + { + "expr": "pg_settings_max_connections", + "legendFormat": "max_connections", + "refId": "B" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 22 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(pg_stat_database_xact_commit{datname=\"postgis\"}[5m])", + "legendFormat": "commits", + "refId": "A" + }, + { + "expr": "rate(pg_stat_database_xact_rollback{datname=\"postgis\"}[5m])", + "legendFormat": "rollbacks", + "refId": "B" + } + ], + "title": "Transaction rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 16, + "y": 22 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "100 * pg_stat_database_blks_hit{datname=\"postgis\"} / clamp_min(pg_stat_database_blks_hit{datname=\"postgis\"} + pg_stat_database_blks_read{datname=\"postgis\"}, 1)", + "legendFormat": "cache hit", + "refId": "A" + } + ], + "title": "Cache hit ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0, + "text": "UP" + }, + "to": 999 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connections" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache hit" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "green", + "value": 99 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 2, + "x": 22, + "y": 22 + }, + "id": 51, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "up{job=\"database\"}", + "legendFormat": "Status", + "refId": "A" + }, + { + "expr": "sum(pg_stat_activity_count{datname=\"postgis\", state=\"active\"})", + "legendFormat": "Connections", + "refId": "B" + }, + { + "expr": "100 * pg_stat_database_blks_hit{datname=\"postgis\"} / clamp_min(pg_stat_database_blks_hit{datname=\"postgis\"} + pg_stat_database_blks_read{datname=\"postgis\"}, 1)", + "legendFormat": "Cache hit", + "refId": "C" + } + ], + "title": "", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "eoapi", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "eoAPI Overview", + "uid": "eoapi-overview", + "version": 6 +} diff --git a/dockerfiles/metrics/grafana/dashboards/eoapi-raster.json b/dockerfiles/metrics/grafana/dashboards/eoapi-raster.json new file mode 100644 index 0000000..32b7258 --- /dev/null +++ b/dockerfiles/metrics/grafana/dashboards/eoapi-raster.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "icon": "dashboard", + "title": "Overview", + "type": "link", + "url": "/d/eoapi-overview/eoapi-overview" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "At a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0, + "text": "UP" + }, + "to": 999 + }, + "type": "range" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "up{job=\"raster\"}", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "sum(rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "Request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"4xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "4xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=\"5xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "5xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m])))", + "refId": "A" + } + ], + "title": "p95 latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 106, + "title": "Errors and latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*5xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*4xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (status) (rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Request rate by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 108, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\", status=~\"4xx|5xx\"}[5m])) or (0 * sum by (operation) (rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Errors by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 109, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by (operation, le) (rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m]))))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Slowest operations (p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 110, + "title": "Traffic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 111, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(titiler_pgstac_http_requests_total{job=\"raster\", operation!=\"\"}[5m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 112, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(titiler_pgstac_http_request_duration_seconds_bucket{job=\"raster\", operation!=\"\"}[5m])))", + "legendFormat": "p95", + "refId": "B" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "eoapi", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "eoAPI \u2014 Raster", + "uid": "eoapi-raster", + "version": 2 +} diff --git a/dockerfiles/metrics/grafana/dashboards/eoapi-stac-auth-proxy.json b/dockerfiles/metrics/grafana/dashboards/eoapi-stac-auth-proxy.json new file mode 100644 index 0000000..195286e --- /dev/null +++ b/dockerfiles/metrics/grafana/dashboards/eoapi-stac-auth-proxy.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "icon": "dashboard", + "title": "Overview", + "type": "link", + "url": "/d/eoapi-overview/eoapi-overview" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "At a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0, + "text": "UP" + }, + "to": 999 + }, + "type": "range" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "up{job=\"stac-auth-proxy\"}", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "Request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\", status=\"4xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "4xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\", status=\"5xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "5xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac-auth-proxy\", operation!=\"\"}[5m])))", + "refId": "A" + } + ], + "title": "p95 latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 106, + "title": "Errors and latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*5xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*4xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (status) (rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Request rate by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 108, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\", status=~\"4xx|5xx\"}[5m])) or (0 * sum by (operation) (rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Errors by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 109, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by (operation, le) (rate(http_request_duration_seconds_bucket{job=\"stac-auth-proxy\", operation!=\"\"}[5m]))))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Slowest operations (p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 110, + "title": "Traffic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 111, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(http_requests_total{job=\"stac-auth-proxy\", operation!=\"\"}[5m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 112, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac-auth-proxy\", operation!=\"\"}[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac-auth-proxy\", operation!=\"\"}[5m])))", + "legendFormat": "p95", + "refId": "B" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "eoapi", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "eoAPI \u2014 STAC Auth Proxy", + "uid": "eoapi-stac-auth-proxy", + "version": 2 +} diff --git a/dockerfiles/metrics/grafana/dashboards/eoapi-stac.json b/dockerfiles/metrics/grafana/dashboards/eoapi-stac.json new file mode 100644 index 0000000..e796976 --- /dev/null +++ b/dockerfiles/metrics/grafana/dashboards/eoapi-stac.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "icon": "dashboard", + "title": "Overview", + "type": "link", + "url": "/d/eoapi-overview/eoapi-overview" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "At a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0, + "text": "UP" + }, + "to": 999 + }, + "type": "range" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "up{job=\"stac\"}", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "Request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac\", operation!=\"\", status=\"4xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "4xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(http_requests_total{job=\"stac\", operation!=\"\", status=\"5xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "5xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac\", operation!=\"\"}[5m])))", + "refId": "A" + } + ], + "title": "p95 latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 106, + "title": "Errors and latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*5xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*4xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (status) (rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Request rate by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 108, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(http_requests_total{job=\"stac\", operation!=\"\", status=~\"4xx|5xx\"}[5m])) or (0 * sum by (operation) (rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Errors by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 109, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by (operation, le) (rate(http_request_duration_seconds_bucket{job=\"stac\", operation!=\"\"}[5m]))))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Slowest operations (p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 110, + "title": "Traffic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 111, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(http_requests_total{job=\"stac\", operation!=\"\"}[5m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 112, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac\", operation!=\"\"}[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"stac\", operation!=\"\"}[5m])))", + "legendFormat": "p95", + "refId": "B" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "eoapi", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "eoAPI \u2014 STAC", + "uid": "eoapi-stac", + "version": 2 +} diff --git a/dockerfiles/metrics/grafana/dashboards/eoapi-vector.json b/dockerfiles/metrics/grafana/dashboards/eoapi-vector.json new file mode 100644 index 0000000..374dd20 --- /dev/null +++ b/dockerfiles/metrics/grafana/dashboards/eoapi-vector.json @@ -0,0 +1,862 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "icon": "dashboard", + "title": "Overview", + "type": "link", + "url": "/d/eoapi-overview/eoapi-overview" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "At a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0, + "text": "UP" + }, + "to": 999 + }, + "type": "range" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "up{job=\"vector\"}", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "Request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "4xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "100 * (sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"5xx\"}[5m])) or on() vector(0)) / clamp_min(sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m])), 0.001)", + "refId": "A" + } + ], + "title": "5xx error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m])))", + "refId": "A" + } + ], + "title": "p95 latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 106, + "title": "Errors and latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*5xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*4xx.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (status) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Request rate by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 108, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m])) or (0 * sum by (operation) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "4xx by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 113, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"5xx\"}[5m])) or (0 * sum by (operation) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "5xx by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 3, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Rate" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Share" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 114, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Rate" + } + ] + }, + "targets": [ + { + "expr": "sum by (operation, method) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m]))", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "expr": "100 * sum by (operation, method) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m])) / clamp_min(sum(rate(tipg_http_requests_total{job=\"vector\", operation!=\"\", status=\"4xx\"}[5m])), 0.001)", + "format": "table", + "instant": true, + "range": false, + "refId": "B" + } + ], + "title": "4xx endpoints", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Rate": 2, + "Share": 3, + "method": 1, + "operation": 0 + }, + "renameByName": { + "Value #A": "Rate", + "Value #B": "Share", + "method": "Method", + "operation": "Operation" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "Rate" + } + ], + "match": "all", + "type": "include" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 109, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by (operation, le) (rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m]))))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Slowest operations (p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 110, + "title": "Traffic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 1, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 111, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (operation) (rate(tipg_http_requests_total{job=\"vector\", operation!=\"\"}[5m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 112, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(tipg_http_request_duration_seconds_bucket{job=\"vector\", operation!=\"\"}[5m])))", + "legendFormat": "p95", + "refId": "B" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "eoapi", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "eoAPI \u2014 Vector", + "uid": "eoapi-vector", + "version": 3 +} diff --git a/dockerfiles/metrics/grafana/provisioning/dashboards/dashboards.yml b/dockerfiles/metrics/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..a1ae4a3 --- /dev/null +++ b/dockerfiles/metrics/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: eoAPI + orgId: 1 + folder: eoAPI + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/dashboards diff --git a/dockerfiles/metrics/grafana/provisioning/datasources/prometheus.yml b/dockerfiles/metrics/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..8704d9e --- /dev/null +++ b/dockerfiles/metrics/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/dockerfiles/metrics/prometheus.yml b/dockerfiles/metrics/prometheus.yml new file mode 100644 index 0000000..95a4512 --- /dev/null +++ b/dockerfiles/metrics/prometheus.yml @@ -0,0 +1,43 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: stac-auth-proxy + metrics_path: /stac/_mgmt/metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: + - stac-auth-proxy:8000 + + - job_name: stac + metrics_path: /_mgmt/metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: + - stac:8081 + + - job_name: raster + metrics_path: /metrics + scrape_interval: 30s + scrape_timeout: 25s + static_configs: + - targets: + - raster:8082 + + - job_name: vector + metrics_path: /metrics + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: + - vector:8083 + + - job_name: database + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: + - postgres-exporter:9187 diff --git a/infrastructure/docker-compose.hetzner.yml b/infrastructure/docker-compose.hetzner.yml index b18213f..dfbf9bd 100644 --- a/infrastructure/docker-compose.hetzner.yml +++ b/infrastructure/docker-compose.hetzner.yml @@ -2,8 +2,9 @@ # # Requires DNS A records for EOAPI_DOMAIN and auth.EOAPI_DOMAIN pointing at the server. # -# COMPOSE_FILE=docker-compose.yml:docker-compose.mock-oidc.yml:infrastructure/docker-compose.hetzner.yml \ +# COMPOSE_FILE=docker-compose.yml:docker-compose.mock-oidc.yml:docker-compose.metrics.yml:infrastructure/docker-compose.hetzner.yml \ # EOAPI_DOMAIN=eoapi.ds.io \ +# EOAPI_GRAFANA_ROOT_URL=https://eoapi.ds.io/monitoring/ \ # ACME_EMAIL=you@example.com \ # docker compose up -d @@ -120,6 +121,31 @@ services: - traefik.http.middlewares.vector-stripprefix.stripprefix.prefixes=/vector - traefik.http.services.vector.loadbalancer.server.port=8083 + grafana: + environment: + GF_AUTH_GENERIC_OAUTH_AUTH_URL: https://auth.${EOAPI_DOMAIN}/authorize + labels: !override + - traefik.enable=true + - traefik.http.routers.grafana.rule=Host(`${EOAPI_DOMAIN}`) && PathPrefix(`/monitoring`) + - traefik.http.routers.grafana.entrypoints=websecure + - traefik.http.routers.grafana.middlewares=forwarded-headers + - traefik.http.routers.grafana.tls.certresolver=letsencrypt + - traefik.http.services.grafana.loadbalancer.server.port=3000 + + traffic-simulator: + image: ghcr.io/astral-sh/uv:python3.12-bookworm-slim + restart: unless-stopped + depends_on: + - traefik + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./scripts:/app/scripts:ro + command: + - bash + - -c + - uv run --with httpx /app/scripts/simulate_traffic_loop.py --base-url https://${EOAPI_DOMAIN} + mock-oidc: ports: !reset [] command: diff --git a/runtimes/eoapi/raster/eoapi/raster/app.py b/runtimes/eoapi/raster/eoapi/raster/app.py index 8e5d565..edb5978 100644 --- a/runtimes/eoapi/raster/eoapi/raster/app.py +++ b/runtimes/eoapi/raster/eoapi/raster/app.py @@ -46,6 +46,7 @@ add_search_list_route, add_search_register_route, ) +from titiler.pgstac.metrics import instrument_app from titiler.pgstac.reader import PgSTACReader from . import __version__ as eoapi_raster_version @@ -578,3 +579,5 @@ def conformance( for prefix in restricted_prefixes ): oidc_auth.apply_auth_dependencies(route, required_token_scopes=[]) + +instrument_app(app) diff --git a/runtimes/eoapi/raster/eoapi/raster/config.py b/runtimes/eoapi/raster/eoapi/raster/config.py index e3db500..5e68660 100644 --- a/runtimes/eoapi/raster/eoapi/raster/config.py +++ b/runtimes/eoapi/raster/eoapi/raster/config.py @@ -70,11 +70,11 @@ def get_postgres_setting(cls, data: Any) -> Any: secret = get_secret_dict(arn) data.update( { - "postgres_host": secret["host"], - "postgres_dbname": secret["dbname"], - "postgres_user": secret["username"], - "postgres_pass": secret["password"], - "postgres_port": secret["port"], + "pghost": secret["host"], + "pgdatabase": secret["dbname"], + "pguser": secret["username"], + "pgpassword": secret["password"], + "pgport": secret["port"], } ) diff --git a/runtimes/eoapi/raster/eoapi/raster/templates/stac-viewer.html b/runtimes/eoapi/raster/eoapi/raster/templates/stac-viewer.html index e372cf6..97a6ea6 100644 --- a/runtimes/eoapi/raster/eoapi/raster/templates/stac-viewer.html +++ b/runtimes/eoapi/raster/eoapi/raster/templates/stac-viewer.html @@ -346,10 +346,8 @@ const set1bViz = () => { params = {} const asset = document.getElementById('asset-selector').value - params.assets = asset - const bidx = document.getElementById('layer-selector').selectedOptions[0].getAttribute("bidx") - params.asset_bidx = `${asset}|${bidx}` + params.assets = `${asset}|bidx=${bidx}` if (['uint8','int8'].indexOf(scope.info[asset].dtype) === -1) { params.rescale = `${document.getElementById('data-min').value},${document.getElementById('data-max').value}` @@ -379,17 +377,14 @@ params = {} let asset_params - let indexes_params if (document.getElementById("compose-switch").checked === true) { - asset_params = [r, g, b].map(i => `assets=${i}`).join('&') - indexes_params = [r, g, b].map(i => `asset_bidx=${i}|1`).join('&') + asset_params = [r, g, b].map(i => `assets=${i}|bidx=1`).join('&') // we can't know the type for merged assets params.rescale = `${document.getElementById('data-min').value},${document.getElementById('data-max').value}` } else { const asset = document.getElementById('asset-selector').value - asset_params = `assets=${asset}` - indexes_params = `asset_bidx=${asset}|${r},${g},${b}` + asset_params = `assets=${asset}|bidx=${r},${g},${b}` if (['uint8','int8'].indexOf(scope.info[asset].dtype) === -1) { params.rescale = `${document.getElementById('data-min').value},${document.getElementById('data-max').value}` @@ -401,7 +396,7 @@ } const url_params = Object.keys(params).map(i => `${i}=${params[i]}`).join('&') - let url = `{{ endpoint }}/WebMercatorQuad/tilejson.json?${url_params}&${asset_params}&${indexes_params}` + let url = `{{ endpoint }}/WebMercatorQuad/tilejson.json?${url_params}&${asset_params}` map.addSource('raster', { type: 'raster', url: url, tileSize: 256 }) map.addLayer({ id: 'raster', type: 'raster', source: 'raster' }) diff --git a/runtimes/eoapi/raster/pyproject.toml b/runtimes/eoapi/raster/pyproject.toml index 0b94437..fb0d934 100644 --- a/runtimes/eoapi/raster/pyproject.toml +++ b/runtimes/eoapi/raster/pyproject.toml @@ -19,8 +19,8 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.pgstac==1.9.0", - "titiler.extensions==0.24.2", + "titiler.pgstac[metrics]==3.1.0", + "titiler.extensions==2.2.0", "starlette-cramjam>=0.4,<0.6", "boto3", "eoapi.auth-utils>=0.2.0", diff --git a/runtimes/eoapi/stac/eoapi/stac/app.py b/runtimes/eoapi/stac/eoapi/stac/app.py index d3a300e..e3f30b9 100644 --- a/runtimes/eoapi/stac/eoapi/stac/app.py +++ b/runtimes/eoapi/stac/eoapi/stac/app.py @@ -6,6 +6,7 @@ import jinja2 from eoapi.auth_utils import OpenIdConnectAuth, OpenIdConnectSettings from fastapi import APIRouter, FastAPI +from stac_fastapi.api.metrics import register_operations from stac_fastapi.api.models import ( CollectionUri, ItemCollectionUri, @@ -229,9 +230,17 @@ async def lifespan(app: FastAPI): search_post_request_model=search_post_model, response_class=JSONResponse, middlewares=middlewares, + add_metrics=True, ) app = api.app +register_operations( + { + ("GET", "/viewer"): "viewer", + ("GET", "/collections/{collection_id}/items/{item_id}/viewer"): "item_viewer", + } +) + @app.get("/viewer", response_class=HTMLResponse) async def viewer_page(request: Request): diff --git a/runtimes/eoapi/stac/pyproject.toml b/runtimes/eoapi/stac/pyproject.toml index ea82bd7..b2ce86b 100644 --- a/runtimes/eoapi/stac/pyproject.toml +++ b/runtimes/eoapi/stac/pyproject.toml @@ -20,14 +20,14 @@ dynamic = ["version"] dependencies = [ "orjson", "stac-fastapi.pgstac>=6.3,<7.0", - "stac-fastapi.api>=6.3,<7.0", - "stac-fastapi.extensions>=6.3,<7.0", - "stac-fastapi.types>=6.3,<7.0", + "stac-fastapi.api[metrics]>=6.5,<7.0", + "stac-fastapi.extensions>=6.5,<7.0", + "stac-fastapi.types>=6.5,<7.0", "jinja2>=2.11.2,<4.0.0", "starlette-cramjam>=0.4,<0.8", "psycopg_pool", "eoapi.auth-utils>=0.2.0", - "boto3" + "boto3", ] [project.optional-dependencies] diff --git a/runtimes/eoapi/vector/eoapi/vector/app.py b/runtimes/eoapi/vector/eoapi/vector/app.py index 8a502da..3d120b9 100644 --- a/runtimes/eoapi/vector/eoapi/vector/app.py +++ b/runtimes/eoapi/vector/eoapi/vector/app.py @@ -17,6 +17,7 @@ from tipg.dependencies import ResponseType, accept_media_type from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers from tipg.factory import Endpoints as TiPgEndpoints +from tipg.metrics import instrument_app from tipg.middleware import CacheControlMiddleware, CatalogUpdateMiddleware from tipg.openapi import _update_openapi from tipg.resources.enums import MediaType @@ -193,3 +194,5 @@ async def refresh(request: Request): for prefix in restricted_prefixes ): oidc_auth.apply_auth_dependencies(route, required_token_scopes=[]) + +instrument_app(app) diff --git a/runtimes/eoapi/vector/pyproject.toml b/runtimes/eoapi/vector/pyproject.toml index 9bd0707..bfb07f6 100644 --- a/runtimes/eoapi/vector/pyproject.toml +++ b/runtimes/eoapi/vector/pyproject.toml @@ -19,9 +19,9 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "tipg==1.2.1", + "tipg[metrics]==1.6.1", "eoapi.auth-utils>=0.2.0", - "boto3" + "boto3", ] [project.optional-dependencies] diff --git a/scripts/demo_db_refresh.sh b/scripts/demo_db_refresh.sh new file mode 100644 index 0000000..0528a04 --- /dev/null +++ b/scripts/demo_db_refresh.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Reset the demo database and reload the standard sample collection. + +set -euo pipefail + +ROOT_DIR="${ROOT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +LOCK_FILE="${EOAPI_DEMO_REFRESH_LOCK_FILE:-/tmp/eoapi-demo-refresh.lock}" + +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml:docker-compose.mock-oidc.yml:docker-compose.metrics.yml:infrastructure/docker-compose.hetzner.yml}" +EOAPI_GRAFANA_ROOT_URL="${EOAPI_GRAFANA_ROOT_URL:-https://${EOAPI_DOMAIN:?set EOAPI_DOMAIN}/monitoring/}" + +export COMPOSE_FILE EOAPI_GRAFANA_ROOT_URL + +cd "$ROOT_DIR" + +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + echo "Another demo refresh is already running; exiting." + exit 0 +fi + +echo "Stopping eoAPI demo stack..." +docker compose down + +echo "Wiping PostgreSQL data directory..." +bash .github/workflows/ingest.sh --wipe-pgdata-only + +echo "Starting eoAPI demo stack..." +docker compose pull --ignore-pull-failures +docker compose up -d --build --remove-orphans + +echo "Loading standard demo data..." +docker compose --profile ingest run --rm ingest + +echo "Restarting vector service after ingest..." +docker compose restart vector + +echo "Demo database refresh complete." diff --git a/scripts/install_demo_cron.sh b/scripts/install_demo_cron.sh new file mode 100644 index 0000000..1a7e13a --- /dev/null +++ b/scripts/install_demo_cron.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Install host cron for refreshing the public demo database at midnight Eastern. + +set -euo pipefail + +DEPLOY_PATH="${DEPLOY_PATH:-$(pwd)}" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml:docker-compose.mock-oidc.yml:docker-compose.metrics.yml:infrastructure/docker-compose.hetzner.yml}" +EOAPI_GRAFANA_ROOT_URL="${EOAPI_GRAFANA_ROOT_URL:-https://${EOAPI_DOMAIN:?set EOAPI_DOMAIN}/monitoring/}" + +: "${ACME_EMAIL:?set ACME_EMAIL}" + +if ! command -v crontab >/dev/null 2>&1; then + echo "crontab is not installed. Install cron before enabling the demo refresh schedule." >&2 + exit 1 +fi + +cron_file="$(mktemp)" +existing_cron="$(mktemp)" +if ! crontab -l > "$existing_cron" 2>/dev/null; then + : > "$existing_cron" +fi + +sed '/# eoapi-demo-refresh begin/,/# eoapi-demo-refresh end/d' \ + "$existing_cron" \ + > "$cron_file" + +cat >> "$cron_file" <> demo-refresh.log 2>&1 +# eoapi-demo-refresh end +CRON + +crontab "$cron_file" +rm "$cron_file" "$existing_cron" + +crontab -l | sed -n '/# eoapi-demo-refresh begin/,/# eoapi-demo-refresh end/p' diff --git a/scripts/simulate_traffic.py b/scripts/simulate_traffic.py new file mode 100755 index 0000000..549b60a --- /dev/null +++ b/scripts/simulate_traffic.py @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +"""Generate realistic traffic against a local eoAPI stack for Grafana demos. + +Emulates typical read-heavy usage across the landing page, STAC Browser paths, +STAC API, raster tiles, and vector features. + +Example: + uv run scripts/simulate_traffic.py + uv run scripts/simulate_traffic.py --duration 5m --users 8 + uv run scripts/simulate_traffic.py --base-url http://127.0.0.1:8080 --duration 180 +""" + +from __future__ import annotations + +import argparse +import asyncio +import random +import re +import sys +import time +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable + +import httpx + +DEFAULT_BASE_URL = "http://127.0.0.1:8080" +DEFAULT_COLLECTION = "noaa-emergency-response" +DEFAULT_ITEM = "20200307aC0853300w361200" +TILE = (15, 8589, 12849) +POINT = (-85.6358, 36.1624) + +BROWSER_HEADERS = { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +} +API_HEADERS = {"Accept": "application/json"} + + +def parse_duration(value: str) -> float: + """Parse seconds or suffix forms like 3m, 90s, 1h.""" + value = value.strip().lower() + match = re.fullmatch(r"(\d+(?:\.\d+)?)([smh])?", value) + if not match: + raise argparse.ArgumentTypeError( + f"invalid duration {value!r}; use seconds or suffix s/m/h" + ) + amount = float(match.group(1)) + unit = match.group(2) or "s" + return amount * {"s": 1, "m": 60, "h": 3600}[unit] + + +@dataclass +class Context: + base_url: str + collection_id: str | None = None + item_ids: list[str] = field(default_factory=list) + search_id: str | None = None + vector_collection: str = "pg_temp.pgstac_collections_view" + vector_item_id: str | None = None + has_stac_data: bool = False + + +@dataclass +class Stats: + ok: int = 0 + errors: int = 0 + by_scenario: dict[str, int] = field(default_factory=dict) + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def record(self, scenario: str, *, success: bool) -> None: + async with self.lock: + self.by_scenario[scenario] = self.by_scenario.get(scenario, 0) + 1 + if success: + self.ok += 1 + else: + self.errors += 1 + + +Scenario = Callable[[httpx.AsyncClient, Context, Stats], Awaitable[None]] + + +async def _request( + client: httpx.AsyncClient, + stats: Stats, + scenario: str, + method: str, + url: str, + *, + expected: set[int] | None = None, + **kwargs: Any, +) -> httpx.Response | None: + expected = expected or {200} + try: + response = await client.request(method, url, **kwargs) + except httpx.HTTPError: + await stats.record(scenario, success=False) + return None + + success = response.status_code in expected + await stats.record(scenario, success=success) + return response + + +async def bootstrap(client: httpx.AsyncClient, ctx: Context) -> None: + """Discover collections, items, and a raster search id from the live stack.""" + response = await client.get( + f"{ctx.base_url}/stac/collections", headers=API_HEADERS, timeout=20 + ) + if response.status_code == 200: + collections = response.json().get("collections") or [] + if collections: + preferred = next( + ( + collection["id"] + for collection in collections + if collection.get("id") == DEFAULT_COLLECTION + ), + collections[0]["id"], + ) + ctx.collection_id = preferred + ctx.has_stac_data = True + + items_resp = await client.get( + f"{ctx.base_url}/stac/collections/{ctx.collection_id}/items", + params={"limit": 10}, + headers=API_HEADERS, + timeout=20, + ) + if items_resp.status_code == 200: + features = items_resp.json().get("features") or [] + ctx.item_ids = [feature["id"] for feature in features[:10]] + + register_body = { + "collections": [ctx.collection_id or DEFAULT_COLLECTION], + "filter-lang": "cql2-json", + } + + register_resp = await client.post( + f"{ctx.base_url}/raster/searches/register", + json=register_body, + headers=API_HEADERS, + timeout=30, + ) + if register_resp.status_code == 200: + ctx.search_id = register_resp.json().get("id") + + vector_resp = await client.get( + f"{ctx.base_url}/vector/collections", headers=API_HEADERS, timeout=20 + ) + if vector_resp.status_code == 200: + collections = vector_resp.json().get("collections") or [] + ids = [collection["id"] for collection in collections] + if "pg_temp.pgstac_collections_view" in ids: + ctx.vector_collection = "pg_temp.pgstac_collections_view" + elif ids: + ctx.vector_collection = ids[0] + + items_resp = await client.get( + f"{ctx.base_url}/vector/collections/{ctx.vector_collection}/items", + params={"limit": 5}, + headers={"Accept": "application/geo+json"}, + timeout=20, + ) + if items_resp.status_code == 200: + features = items_resp.json().get("features") or [] + if features: + ctx.vector_item_id = features[0]["id"] + + +async def scenario_landing( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + await _request( + client, + stats, + "web.landing", + "GET", + f"{ctx.base_url}/", + headers=BROWSER_HEADERS, + ) + await _request( + client, + stats, + "web.assets", + "GET", + f"{ctx.base_url}/assets/home.css", + headers={"Accept": "text/css,*/*;q=0.1"}, + ) + await _request( + client, + stats, + "web.assets", + "GET", + f"{ctx.base_url}/img/eoapi_logo_small.png", + headers={"Accept": "image/*,*/*;q=0.8"}, + ) + + +async def scenario_browser( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + await _request( + client, + stats, + "web.browser", + "GET", + f"{ctx.base_url}/browser/", + headers=BROWSER_HEADERS, + ) + await _request( + client, + stats, + "web.stac_viewer", + "GET", + f"{ctx.base_url}/stac/viewer", + headers=BROWSER_HEADERS, + ) + await _request( + client, + stats, + "web.mosaic_builder", + "GET", + f"{ctx.base_url}/raster/searches/builder", + headers=BROWSER_HEADERS, + ) + + +async def scenario_stac_catalog( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + await _request( + client, + stats, + "stac.landing", + "GET", + f"{ctx.base_url}/stac/", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "stac.conformance", + "GET", + f"{ctx.base_url}/stac/conformance", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "stac.collections", + "GET", + f"{ctx.base_url}/stac/collections", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "stac.queryables", + "GET", + f"{ctx.base_url}/stac/queryables", + headers=API_HEADERS, + ) + if not ctx.has_stac_data or not ctx.collection_id: + return + + await _request( + client, + stats, + "stac.collection", + "GET", + f"{ctx.base_url}/stac/collections/{ctx.collection_id}", + headers=API_HEADERS, + ) + + +async def scenario_stac_items( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + if not ctx.has_stac_data or not ctx.collection_id: + return + + item_id = random.choice(ctx.item_ids) if ctx.item_ids else DEFAULT_ITEM + item_expected = {200} if ctx.item_ids else {200, 404} + await _request( + client, + stats, + "stac.items", + "GET", + f"{ctx.base_url}/stac/collections/{ctx.collection_id}/items", + params={"limit": random.choice([1, 5, 10])}, + headers=API_HEADERS, + ) + await _request( + client, + stats, + "stac.item", + "GET", + f"{ctx.base_url}/stac/collections/{ctx.collection_id}/items/{item_id}", + headers=API_HEADERS, + expected=item_expected, + ) + await _request( + client, + stats, + "stac.item_viewer", + "GET", + f"{ctx.base_url}/stac/collections/{ctx.collection_id}/items/{item_id}/viewer", + headers=BROWSER_HEADERS, + expected={200, 307, 308, 404}, + ) + + +async def scenario_stac_search( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + params: dict[str, Any] = {"limit": random.choice([1, 5, 10])} + body: dict[str, Any] = {"limit": random.choice([1, 5, 10])} + if ctx.collection_id: + params["collections"] = ctx.collection_id + body["collections"] = [ctx.collection_id] + + await _request( + client, + stats, + "stac.search_get", + "GET", + f"{ctx.base_url}/stac/search", + params=params, + headers=API_HEADERS, + ) + await _request( + client, + stats, + "stac.search_post", + "POST", + f"{ctx.base_url}/stac/search", + json=body, + headers=API_HEADERS, + ) + + +async def scenario_raster( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + z, x, y = TILE + lon, lat = POINT + await _request( + client, + stats, + "raster.health", + "GET", + f"{ctx.base_url}/raster/healthz", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "raster.landing", + "GET", + f"{ctx.base_url}/raster/", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "raster.searches", + "GET", + f"{ctx.base_url}/raster/searches", + headers=API_HEADERS, + ) + + if ctx.search_id: + await _request( + client, + stats, + "raster.search_info", + "GET", + f"{ctx.base_url}/raster/searches/{ctx.search_id}/info", + headers=API_HEADERS, + timeout=30, + ) + await _request( + client, + stats, + "raster.search_tile_assets", + "GET", + f"{ctx.base_url}/raster/searches/{ctx.search_id}/tiles/WebMercatorQuad/{z}/{x}/{y}/assets", + headers=API_HEADERS, + timeout=30, + expected={200, 404}, + ) + await _request( + client, + stats, + "raster.search_tile", + "GET", + f"{ctx.base_url}/raster/searches/{ctx.search_id}/tiles/WebMercatorQuad/{z}/{x}/{y}", + params={"assets": "cog"}, + headers={"Accept": "image/jpeg,image/png,*/*"}, + timeout=60, + expected={200, 204, 404}, + ) + + if not ctx.has_stac_data or not ctx.collection_id: + return + + await _request( + client, + stats, + "raster.point", + "GET", + f"{ctx.base_url}/raster/collections/{ctx.collection_id}/point/{lon},{lat}/assets", + headers=API_HEADERS, + timeout=30, + expected={200, 404}, + ) + await _request( + client, + stats, + "raster.tile_assets", + "GET", + f"{ctx.base_url}/raster/collections/{ctx.collection_id}/tiles/WebMercatorQuad/{z}/{x}/{y}/assets", + headers=API_HEADERS, + timeout=30, + expected={200, 404}, + ) + await _request( + client, + stats, + "raster.tile", + "GET", + f"{ctx.base_url}/raster/collections/{ctx.collection_id}/tiles/WebMercatorQuad/{z}/{x}/{y}", + params={"assets": "cog"}, + headers={"Accept": "image/jpeg,image/png,*/*"}, + timeout=60, + expected={200, 404}, + ) + + if ctx.item_ids: + item_id = random.choice(ctx.item_ids) + await _request( + client, + stats, + "raster.item_tilejson", + "GET", + f"{ctx.base_url}/raster/collections/{ctx.collection_id}/items/{item_id}/WebMercatorQuad/tilejson.json", + headers=API_HEADERS, + timeout=30, + expected={200, 404}, + ) + + +async def scenario_vector( + client: httpx.AsyncClient, ctx: Context, stats: Stats +) -> None: + await _request( + client, + stats, + "vector.landing", + "GET", + f"{ctx.base_url}/vector/", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "vector.conformance", + "GET", + f"{ctx.base_url}/vector/conformance", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "vector.collections", + "GET", + f"{ctx.base_url}/vector/collections", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "vector.collection", + "GET", + f"{ctx.base_url}/vector/collections/{ctx.vector_collection}", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "vector.items", + "GET", + f"{ctx.base_url}/vector/collections/{ctx.vector_collection}/items", + params={"limit": random.choice([1, 5, 10])}, + headers={"Accept": "application/geo+json"}, + ) + if ctx.vector_item_id: + await _request( + client, + stats, + "vector.item", + "GET", + f"{ctx.base_url}/vector/collections/{ctx.vector_collection}/items/{ctx.vector_item_id}", + headers={"Accept": "application/geo+json"}, + ) + await _request( + client, + stats, + "vector.tilejson", + "GET", + f"{ctx.base_url}/vector/collections/{ctx.vector_collection}/tiles/WebMercatorQuad/tilejson.json", + headers=API_HEADERS, + ) + await _request( + client, + stats, + "vector.tile", + "GET", + f"{ctx.base_url}/vector/collections/public.my_data/tiles/WebMercatorQuad/0/0/0", + headers={"Accept": "application/vnd.mapbox-vector-tile,*/*"}, + expected={200, 404}, + ) + + +SCENARIOS: list[tuple[int, Scenario]] = [ + (12, scenario_landing), + (10, scenario_browser), + (18, scenario_stac_catalog), + (20, scenario_stac_items), + (15, scenario_stac_search), + (18, scenario_raster), + (7, scenario_vector), +] + + +async def user_loop( + user_id: int, + client: httpx.AsyncClient, + ctx: Context, + stats: Stats, + stop_at: float, + think_min: float, + think_max: float, +) -> None: + weights, scenarios = zip(*SCENARIOS) + while time.monotonic() < stop_at: + scenario = random.choices(scenarios, weights=weights, k=1)[0] + await scenario(client, ctx, stats) + await asyncio.sleep(random.uniform(think_min, think_max)) + + +async def reporter(stats: Stats, stop_at: float, interval: float) -> None: + start = time.monotonic() + while time.monotonic() < stop_at: + await asyncio.sleep(interval) + elapsed = time.monotonic() - start + async with stats.lock: + total = stats.ok + stats.errors + rate = total / elapsed if elapsed else 0.0 + print( + f"[{elapsed:6.0f}s] requests={total:5d} ok={stats.ok:5d} " + f"errors={stats.errors:3d} rate={rate:5.1f}/s", + flush=True, + ) + + +async def run(args: argparse.Namespace) -> int: + base_url = args.base_url.rstrip("/") + duration = args.duration + stop_at = time.monotonic() + duration + + stats = Stats() + ctx = Context(base_url=base_url) + + limits = httpx.Limits( + max_connections=args.users * 4, max_keepalive_connections=args.users * 2 + ) + timeout = httpx.Timeout(args.timeout, connect=10.0) + + async with httpx.AsyncClient( + limits=limits, timeout=timeout, follow_redirects=True + ) as client: + print(f"Bootstrapping against {base_url} ...", flush=True) + try: + await bootstrap(client, ctx) + except httpx.HTTPError as exc: + print(f"Bootstrap warning: {exc}", file=sys.stderr) + + print( + f"Using collection={ctx.collection_id!r}, items={len(ctx.item_ids)}, " + f"search_id={ctx.search_id!r}, stac_data={ctx.has_stac_data}", + flush=True, + ) + print( + f"Running {args.users} virtual users for {duration:.0f}s " + f"(think {args.think_min:.1f}-{args.think_max:.1f}s between scenarios)", + flush=True, + ) + + tasks = [ + asyncio.create_task( + user_loop( + user_id, + client, + ctx, + stats, + stop_at, + args.think_min, + args.think_max, + ) + ) + for user_id in range(args.users) + ] + tasks.append( + asyncio.create_task(reporter(stats, stop_at, args.report_interval)) + ) + + await asyncio.gather(*tasks) + + elapsed = duration + total = stats.ok + stats.errors + print("\nDone.", flush=True) + print( + f"Total requests: {total} ({stats.ok} ok, {stats.errors} errors) " + f"over {elapsed:.0f}s ({total / elapsed:.1f}/s avg)", + flush=True, + ) + print("Top scenarios:", flush=True) + for name, count in sorted( + stats.by_scenario.items(), key=lambda item: item[1], reverse=True + )[:12]: + print(f" {name:22s} {count}", flush=True) + + return 1 if stats.errors and stats.ok == 0 else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Simulate typical eoAPI traffic for Grafana dashboard demos." + ) + parser.add_argument( + "--base-url", + default=DEFAULT_BASE_URL, + help=f"eoAPI base URL via Traefik (default: {DEFAULT_BASE_URL})", + ) + parser.add_argument( + "--duration", + type=parse_duration, + default=parse_duration("3m"), + help="Run time in seconds or with suffix s/m/h (default: 3m)", + ) + parser.add_argument( + "--users", + type=int, + default=6, + help="Concurrent virtual users (default: 6)", + ) + parser.add_argument( + "--think-min", + type=float, + default=0.4, + help="Minimum pause between scenario batches per user (default: 0.4s)", + ) + parser.add_argument( + "--think-max", + type=float, + default=2.5, + help="Maximum pause between scenario batches per user (default: 2.5s)", + ) + parser.add_argument( + "--timeout", + type=float, + default=45.0, + help="Per-request timeout in seconds (default: 45)", + ) + parser.add_argument( + "--report-interval", + type=float, + default=15.0, + help="Progress report interval in seconds (default: 15)", + ) + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + if args.users < 1: + parser.error("--users must be at least 1") + if args.think_min < 0 or args.think_max < args.think_min: + parser.error("--think-min/--think-max are invalid") + + try: + raise SystemExit(asyncio.run(run(args))) + except KeyboardInterrupt: + print("\nInterrupted.", flush=True) + raise SystemExit(130) from None + + +if __name__ == "__main__": + main() diff --git a/scripts/simulate_traffic_loop.py b/scripts/simulate_traffic_loop.py new file mode 100644 index 0000000..0b2ee8b --- /dev/null +++ b/scripts/simulate_traffic_loop.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Run eoAPI traffic simulations continuously with randomized intensity.""" + +from __future__ import annotations + +import argparse +import asyncio +import random +import time +from dataclasses import dataclass + +import simulate_traffic + + +@dataclass(frozen=True) +class TrafficProfile: + name: str + weight: int + users: tuple[int, int] + duration: tuple[int, int] + think_min: tuple[float, float] + think_max: tuple[float, float] + pause: tuple[int, int] + + +PROFILES = [ + TrafficProfile( + "idle", 8, (1, 2), (3 * 60, 8 * 60), (3.0, 8.0), (10.0, 25.0), (60, 5 * 60) + ), + TrafficProfile( + "easy", 35, (2, 5), (5 * 60, 15 * 60), (1.0, 4.0), (4.0, 12.0), (20, 2 * 60) + ), + TrafficProfile( + "normal", 35, (5, 10), (5 * 60, 20 * 60), (0.4, 1.5), (2.5, 6.0), (10, 90) + ), + TrafficProfile( + "heavy", 17, (10, 18), (3 * 60, 10 * 60), (0.1, 0.5), (0.8, 2.5), (10, 60) + ), + TrafficProfile( + "burst", 5, (18, 30), (60, 3 * 60), (0.0, 0.2), (0.3, 1.2), (30, 3 * 60) + ), +] + + +def choose_profile() -> TrafficProfile: + return random.choices( + PROFILES, weights=[profile.weight for profile in PROFILES], k=1 + )[0] + + +def random_range(bounds: tuple[int, int] | tuple[float, float]) -> float: + low, high = bounds + return random.uniform(low, high) + + +async def run_forever(args: argparse.Namespace) -> None: + base_url = args.base_url.rstrip("/") + + while True: + profile = choose_profile() + users = random.randint(*profile.users) + duration = random.randint(*profile.duration) + think_min = random_range(profile.think_min) + think_max = max(think_min, random_range(profile.think_max)) + pause = random.randint(*profile.pause) + + print( + f"Starting {profile.name} traffic: users={users}, duration={duration}s, " + f"think={think_min:.1f}-{think_max:.1f}s, base_url={base_url}", + flush=True, + ) + + run_args = argparse.Namespace( + base_url=base_url, + duration=float(duration), + users=users, + think_min=think_min, + think_max=think_max, + timeout=args.timeout, + report_interval=args.report_interval, + ) + started = time.monotonic() + try: + await simulate_traffic.run(run_args) + except Exception as exc: # noqa: BLE001 - keep the long-running loop alive. + print(f"Traffic run failed: {exc!r}", flush=True) + + elapsed = time.monotonic() - started + print( + f"Finished {profile.name} traffic after {elapsed:.0f}s; " + f"sleeping {pause}s before next run.", + flush=True, + ) + await asyncio.sleep(pause) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Continuously simulate randomized eoAPI demo traffic." + ) + parser.add_argument( + "--base-url", + default=simulate_traffic.DEFAULT_BASE_URL, + help=f"eoAPI base URL (default: {simulate_traffic.DEFAULT_BASE_URL})", + ) + parser.add_argument( + "--timeout", + type=float, + default=45.0, + help="Per-request timeout in seconds (default: 45)", + ) + parser.add_argument( + "--report-interval", + type=float, + default=60.0, + help="Progress report interval in seconds (default: 60)", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + try: + asyncio.run(run_forever(args)) + except KeyboardInterrupt: + print("\nInterrupted.", flush=True) + + +if __name__ == "__main__": + main()