diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fd0cb237..0956ae52 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,7 +21,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} TM_FRONTEND_CERT_ARN: ${{ secrets.TM_FRONTEND_CERT_ARN }} TM_LABS_WILDCARD_CERT_ARN: ${{ secrets.TM_LABS_WILDCARD_CERT_ARN }} - MBTA_V2_API_KEY: ${{ secrets.MBTA_V2_API_KEY }} + MBTA_V3_API_KEY: ${{ secrets.MBTA_V3_API_KEY }} DD_API_KEY: ${{ secrets.DD_API_KEY }} steps: diff --git a/README.md b/README.md index e283b13f..d9a21d4e 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,7 @@ This is the repository for the TransitMatters Data Dashboard. Client code is wri ## Development Instructions -1. Add `MBTA_V2_API_KEY` and `MBTA_V3_API_KEY` to your shell environment: - - `export MBTA_V2_API_KEY='KEY'` in ~/.bashrc or ~/.zshrc +1. Add `MBTA_V3_API_KEY` to your shell environment: - `export MBTA_V3_API_KEY='KEY'` in ~/.bashrc or ~/.zshrc 2. Add your AWS credentials (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) to your shell environment, OR add them to a .boto config file with awscli command `aws configure`. 3. In the root directory, run `npm install` to install all frontend and backend dependencies diff --git a/common/components/charts/SingleDayLineChart.tsx b/common/components/charts/SingleDayLineChart.tsx index 9d8e0913..d4460702 100644 --- a/common/components/charts/SingleDayLineChart.tsx +++ b/common/components/charts/SingleDayLineChart.tsx @@ -16,6 +16,7 @@ import { watermarkLayout } from '../../constants/charts'; import { writeError } from '../../utils/chartError'; import { getFormattedTimeString } from '../../utils/time'; import { AlertsDisclaimer } from '../general/AlertsDisclaimer'; +import { FIVE_MINUTES } from '../../constants/time'; import { LegendSingleDay } from './Legend'; import { ChartDiv } from './ChartDiv'; import { ChartBorder } from './ChartBorder'; @@ -93,14 +94,14 @@ export const SingleDayLineChart: React.FC = ({ // Format benchmark data if it exists. const benchmarkData = data.map((datapoint) => - benchmarkField && datapoint[benchmarkField] ? datapoint[benchmarkField] : undefined + benchmarkField && datapoint[benchmarkField] ? datapoint[benchmarkField] : null ); - const displayBenchmarkData = benchmarkData.every((datapoint) => datapoint !== undefined); - // Have to use `as number` because typescript doesn't understand `datapoint` is not undefined. + const displayBenchmarkData = benchmarkData.some((datapoint) => datapoint !== null); + const multiplier = units === 'Minutes' ? 1 / 60 : 1; - const benchmarkDataFormatted = displayBenchmarkData - ? benchmarkData.map((datapoint) => ((datapoint as number) * multiplier).toFixed(2)) - : null; + const benchmarkDataFormatted = benchmarkData + .map((datapoint) => (datapoint ? (datapoint * multiplier).toFixed(2) : null)) + .filter((datapoint) => datapoint !== null); const convertedData = data.map((datapoint) => ((datapoint[metricField] as number) * multiplier).toFixed(2) @@ -225,7 +226,7 @@ export const SingleDayLineChart: React.FC = ({ const today = new Date(`${date}T00:00:00`); const low = new Date(today); low.setHours(6); - axis.min = Math.min(axis.min, low.valueOf()); + axis.min = Math.min(axis.min - FIVE_MINUTES, low.valueOf()); const high = new Date(today); high.setDate(high.getDate() + 1); high.setHours(1); diff --git a/common/types/charts.ts b/common/types/charts.ts index 0f6f0e6a..5df1e091 100644 --- a/common/types/charts.ts +++ b/common/types/charts.ts @@ -11,7 +11,7 @@ export interface SingleDayDataPoint { headway_time_sec?: number; dwell_time_sec?: number; benchmark_travel_time_sec?: number; - benchmark_headway_time_sec?: number; + benchmark_headway_time_sec?: number | null; threshold_flag_1?: string; speed_mph?: number; benchmark_speed_mph?: number; diff --git a/deploy.sh b/deploy.sh index b4ade1c1..8efff49c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -26,8 +26,8 @@ while getopts "pc" opt; do done # Ensure required secrets are set -if [[ -z "$MBTA_V2_API_KEY" || -z "$DD_API_KEY" ]]; then - echo "Must provide MBTA_V2_API_KEY and DD_API_KEY in environment to deploy" 1>&2 +if [[ -z "$MBTA_V3_API_KEY" || -z "$DD_API_KEY" ]]; then + echo "Must provide MBTA_V3_API_KEY and DD_API_KEY in environment to deploy" 1>&2 exit 1 elif [ -z "$TM_FRONTEND_CERT_ARN" ] && [ -z "$TM_LABS_WILDCARD_CERT_ARN" ]; then echo "Must provide TM_FRONTEND_CERT_ARN or TM_LABS_WILDCARD_CERT_ARN in environment to deploy" 1>&2 @@ -98,7 +98,7 @@ aws cloudformation deploy --template-file cfn/packaged.yaml --stack-name $CF_STA TMBackendCertArn=$BACKEND_CERT_ARN \ TMBackendHostname=$BACKEND_HOSTNAME \ TMBackendZone=$BACKEND_ZONE \ - MbtaV2ApiKey=$MBTA_V2_API_KEY \ + MbtaV3ApiKey=$MBTA_V3_API_KEY \ DDApiKey=$DD_API_KEY \ GitVersion=$GIT_VERSION \ DDTags=$DD_TAGS diff --git a/package-lock.json b/package-lock.json index df1a208e..751b793a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "t-performance-dash", - "version": "4.0.0", + "version": "4.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "t-performance-dash", - "version": "4.0.0", + "version": "4.1.0", "hasInstallScript": true, "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.5.2", @@ -16,8 +16,8 @@ "@headlessui/react": "^1.7.19", "@heroicons/react": "^2.1.3", "@svgr/webpack": "^8.1.0", - "@tanstack/react-query": "^5.29.2", - "@tanstack/react-query-devtools": "^5.29.2", + "@tanstack/react-query": "^5.32.0", + "@tanstack/react-query-devtools": "^5.32.0", "@tippyjs/react": "^4.2.6", "@types/react-flatpickr": "^3.8.11", "bezier-js": "^6.1.4", @@ -2119,70 +2119,6 @@ "react": ">=16.8.0" } }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz", - "integrity": "sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz", - "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz", - "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz", - "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@esbuild/darwin-arm64": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz", @@ -2199,294 +2135,6 @@ "node": ">=12" } }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz", - "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz", - "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz", - "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz", - "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz", - "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz", - "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz", - "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz", - "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz", - "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz", - "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz", - "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz", - "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz", - "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz", - "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz", - "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz", - "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz", - "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz", - "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -3023,126 +2671,6 @@ "node": ">= 10" } }, - "node_modules/@next/swc-darwin-x64": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.1.tgz", - "integrity": "sha512-dAdWndgdQi7BK2WSXrx4lae7mYcOYjbHJUhvOUnJjMNYrmYhxbbvJ2xElZpxNxdfA6zkqagIB9He2tQk+l16ew==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.1.tgz", - "integrity": "sha512-2ZctfnyFOGvTkoD6L+DtQtO3BfFz4CapoHnyLTXkOxbZkVRgg3TQBUjTD/xKrO1QWeydeo8AWfZRg8539qNKrg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.1.tgz", - "integrity": "sha512-jazZXctiaanemy4r+TPIpFP36t1mMwWCKMsmrTRVChRqE6putyAxZA4PDujx0SnfvZHosjdkx9xIq9BzBB5tWg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.1.tgz", - "integrity": "sha512-VjCHWCjsAzQAAo8lkBOLEIkBZFdfW+Z18qcQ056kL4KpUYc8o59JhLDCBlhg+hINQRgzQ2UPGma2AURGOH0+Qg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.1.tgz", - "integrity": "sha512-7HZKYKvAp4nAHiHIbY04finRqjeYvkITOGOurP1aLMexIFG/1+oCnqhGogBdc4lao/lkMW1c+AkwWSzSlLasqw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.1.tgz", - "integrity": "sha512-YGHklaJ/Cj/F0Xd8jxgj2p8po4JTCi6H7Z3Yics3xJhm9CPIqtl8erlpK1CLv+HInDqEWfXilqatF8YsLxxA2Q==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.1.tgz", - "integrity": "sha512-o+ISKOlvU/L43ZhtAAfCjwIfcwuZstiHVXq/BDsZwGqQE0h/81td95MPHliWCnFoikzWcYqh+hz54ZB2FIT8RA==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.2.1", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.1.tgz", - "integrity": "sha512-GmRoTiLcvCLifujlisknv4zu9/C4i9r0ktsA8E51EMqJL4bD4CpO7lDYr7SrUxCR0tS4RVcrqKmCak24T0ohaw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5183,9 +4711,9 @@ } }, "node_modules/@tanstack/query-core": { - "version": "5.29.0", - "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.29.0.tgz", - "integrity": "sha512-WgPTRs58hm9CMzEr5jpISe8HXa3qKQ8CxewdYZeVnA54JrPY9B1CZiwsCoLpLkf0dGRZq+LcX5OiJb0bEsOFww==", + "version": "5.32.0", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.32.0.tgz", + "integrity": "sha512-Z3flEgCat55DRXU5UMwYU1U+DgFZKA3iufyOKs+II7iRAo0uXkeU7PH5e6sOH1CGEag0IpKmZxlUFpCg6roSKw==", "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" @@ -5201,11 +4729,11 @@ } }, "node_modules/@tanstack/react-query": { - "version": "5.29.2", - "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.29.2.tgz", - "integrity": "sha512-nyuWILR4u7H5moLGSiifLh8kIqQDLNOHGuSz0rcp+J75fNc8aQLyr5+I2JCHU3n+nJrTTW1ssgAD8HiKD7IFBQ==", + "version": "5.32.0", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.32.0.tgz", + "integrity": "sha512-+E3UudQtarnx9A6xhpgMZapyF+aJfNBGFMgI459FnduEZqT/9KhOWnMOneZahLRt52yzskSA0AuOyLkXHK0yBA==", "dependencies": { - "@tanstack/query-core": "5.29.0" + "@tanstack/query-core": "5.32.0" }, "funding": { "type": "github", @@ -5216,9 +4744,9 @@ } }, "node_modules/@tanstack/react-query-devtools": { - "version": "5.29.2", - "resolved": "https://registry.npmjs.org/@tanstack/react-query-devtools/-/react-query-devtools-5.29.2.tgz", - "integrity": "sha512-EmsaLNa8iFtReAW+5ftom0/TW78fIosVor517ak/+JFaoTBw8Yub3ao937JFE6AM3K/HXhteqvObetgt1ndLcw==", + "version": "5.32.0", + "resolved": "https://registry.npmjs.org/@tanstack/react-query-devtools/-/react-query-devtools-5.32.0.tgz", + "integrity": "sha512-KWrzLoUjs9JtDSH3H2qbm5MjjykyAT8DkvP8tukw3gBG4ziu5WaWHciBjMsYSe1JB79AOxxGovzjW/Cd9+ofVw==", "dependencies": { "@tanstack/query-devtools": "5.28.10" }, @@ -5227,7 +4755,7 @@ "url": "https://github.com/sponsors/tannerlinsley" }, "peerDependencies": { - "@tanstack/react-query": "^5.29.2", + "@tanstack/react-query": "^5.32.0", "react": "^18.0.0" } }, diff --git a/package.json b/package.json index bfa259be..820456cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "t-performance-dash", - "version": "4.0.0", + "version": "4.1.0", "private": true, "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.5.2", @@ -10,8 +10,8 @@ "@headlessui/react": "^1.7.19", "@heroicons/react": "^2.1.3", "@svgr/webpack": "^8.1.0", - "@tanstack/react-query": "^5.29.2", - "@tanstack/react-query-devtools": "^5.29.2", + "@tanstack/react-query": "^5.32.0", + "@tanstack/react-query-devtools": "^5.32.0", "@tippyjs/react": "^4.2.6", "@types/react-flatpickr": "^3.8.11", "bezier-js": "^6.1.4", diff --git a/server/app.py b/server/app.py index 6d1e1f68..661e37c0 100644 --- a/server/app.py +++ b/server/app.py @@ -2,12 +2,11 @@ import os import subprocess from chalice import Chalice, CORSConfig, ConflictError, Response, ConvertToMiddleware -from datetime import date, timedelta +from datetime import date from datadog_lambda.wrapper import datadog_lambda_wrapper from chalicelib import ( aggregation, data_funcs, - MbtaPerformanceAPI, secrets, mbta_v3, speed, @@ -30,7 +29,7 @@ app.register_middleware(ConvertToMiddleware(datadog_lambda_wrapper)) -def parse_user_date(user_date): +def parse_user_date(user_date: str): date_split = user_date.split("-") [year, month, day] = [int(x) for x in date_split[0:3]] return date(year=year, month=month, day=day) @@ -47,19 +46,11 @@ def mutlidict_to_dict(mutlidict): def healthcheck(): # These functions must return True or False :-) checks = { - "API Key Present": (lambda: len(secrets.MBTA_V2_API_KEY) > 0), + "API Key Present": (lambda: len(secrets.MBTA_V3_API_KEY) > 0), "S3 Headway Fetching": ( lambda: "2020-11-07T10:33:40" in json.dumps(data_funcs.headways(date(year=2020, month=11, day=7), ["70061"])) ), - "Performance API Check": ( - lambda: MbtaPerformanceAPI.get_api_data( - "headways", - {"stop": [70067]}, - date.today() - timedelta(days=1), - date.today(), - ) - ), } failed_checks = {} diff --git a/server/chalicelib/MbtaPerformanceAPI.py b/server/chalicelib/MbtaPerformanceAPI.py deleted file mode 100644 index eafb87ec..00000000 --- a/server/chalicelib/MbtaPerformanceAPI.py +++ /dev/null @@ -1,119 +0,0 @@ -import json -import datetime -import pytz -import requests -from urllib.parse import urlencode -from decimal import Decimal -import itertools - -from chalicelib.parallel import make_parallel - -from .secrets import MBTA_V2_API_KEY - - -def get_timestamps(day): - # has to start after 3:30am, east coast time - bos_tz = pytz.timezone("America/New_York") - start_time = datetime.time(3, 30, 1) - - # build from and to datetimes for a single day - from_dt = bos_tz.localize(datetime.datetime.combine(day, start_time)) - to_dt = from_dt + datetime.timedelta(days=1, seconds=-1) - - # build dict to pass to next - dt_str = {} - dt_str["from_dt_str"] = str(int(from_dt.timestamp())) - dt_str["to_dt_str"] = str(int(to_dt.timestamp())) - return dt_str - - -def get_timestamp_range(start_day, end_day=None): - if end_day: - a = get_timestamps(start_day) - z = get_timestamps(end_day) - dt_str = {} - dt_str["from_dt_str"] = a["from_dt_str"] - dt_str["to_dt_str"] = z["to_dt_str"] - return dt_str - else: - return get_timestamps(start_day) - - -def get_single_url(start_day, end_day, module, params): - # import api key & set base url - base_url_v2 = "https://realtime.mbta.com/developer/api/v2.1/{command}?{parameters}" - - # get datetimes - dt_str = get_timestamp_range(start_day, end_day) - - # format parameters - params["format"] = "json" - params["api_key"] = MBTA_V2_API_KEY - params["from_datetime"] = dt_str.get("from_dt_str") - params["to_datetime"] = dt_str.get("to_dt_str") - - # build url - url = base_url_v2.format(command=module, parameters=urlencode(params)) - return url - - -def get_product_of_list_dict_values(dict_of_lists): - keys = dict_of_lists.keys() - values = dict_of_lists.values() - for combination in itertools.product(*values): - yield dict(zip(keys, combination)) - - -def get_many_urls(start_day, end_day, module, params): - exploded_params = list(get_product_of_list_dict_values(params)) # get all possible parameter combinations - url_list = [] - # get url for each pair, add to list - for single_param in exploded_params: - url_list.append(get_single_url(start_day, end_day, module, single_param)) - return url_list - - -def get_single_api_data(url): - print("Requesting data from URL: {}".format(url)) - response = requests.get(url) - try: - response.raise_for_status() - except requests.exceptions.HTTPError: - print(response.content.decode("utf-8")) - raise # TODO: catch this gracefully - data = json.loads(response.content.decode("utf-8"), parse_float=Decimal, parse_int=Decimal) - return data - - -# This is the primary function, but shouldn't be called directly, see dispatcher below -def _get_api_data(date_interval, module, params): - start_day, end_day = date_interval - url_list = get_many_urls(start_day, end_day, module, params) - all_data = [] - for url in url_list: - data = get_single_api_data(url=url) - all_data.append(data) - return all_data - - -_multithreaded_api = make_parallel(_get_api_data) - - -# we offer this convenient wrapper, that also dispatches to multi-threaded if needed -def get_api_data(module, params, start_day, end_day=None): - if end_day is None: - return _get_api_data((start_day, None), module, params) - else: - return _multithreaded_api(get_7day_chunks(start_day, end_day), module, params) - - -# MBTA api won't accept queries > 7 days. 6 day interval here because of DST. -# this function operates on datetime.dates -def get_7day_chunks(start, end): - delta = (end - start).days + 1 - cur = start - while delta != 0: - inc = min(delta, 6) # Stupid DST hour throws us over if we actually use 7. - yield (cur, cur + datetime.timedelta(days=inc - 1)) - delta -= inc - cur += datetime.timedelta(days=inc) diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index 7483ac4e..810a2f1c 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -54,7 +54,7 @@ def faster_describe(grouped): # `travel_times_over_time` is legacy and returns just the by_date aggregation w/ peak == all -def aggregate_traveltime_data(start_date: str | datetime.date, end_date: str | datetime.date, from_stops, to_stops): +def aggregate_traveltime_data(start_date: datetime.date, end_date: datetime.date, from_stops, to_stops): all_data = data_funcs.travel_times(start_date, from_stops, to_stops, end_date) if not all_data: return None @@ -99,7 +99,7 @@ def calc_travel_times_by_date(df): return summary_stats_final -def travel_times_all(start_date: str | datetime.date, end_date: str, from_stops, to_stops): +def travel_times_all(start_date: datetime.date, end_date: datetime.date, from_stops, to_stops): df = aggregate_traveltime_data(start_date, end_date, from_stops, to_stops) if df is None: return {"by_date": [], "by_time": []} @@ -112,7 +112,7 @@ def travel_times_all(start_date: str | datetime.date, end_date: str, from_stops, } -def travel_times_over_time(start_date: str | datetime.date, end_date: str | datetime.date, from_stops, to_stops): +def travel_times_over_time(start_date: datetime.date, end_date: datetime.date, from_stops, to_stops): df = aggregate_traveltime_data(start_date, end_date, from_stops, to_stops) if df is None: return [] @@ -123,7 +123,7 @@ def travel_times_over_time(start_date: str | datetime.date, end_date: str | date #################### # HEADWAYS #################### -def headways_over_time(start_date: str | datetime.date, end_date: str | datetime.date, stops): +def headways_over_time(start_date: datetime.date, end_date: datetime.date, stops): all_data = data_funcs.headways(start_date, stops, end_date) if not all_data: return [] diff --git a/server/chalicelib/data_funcs.py b/server/chalicelib/data_funcs.py index 63d88da6..c3e48800 100644 --- a/server/chalicelib/data_funcs.py +++ b/server/chalicelib/data_funcs.py @@ -3,10 +3,11 @@ import traceback from datetime import date, timedelta from typing import Dict, Any, Callable, List, Union -from chalicelib import MbtaPerformanceAPI, s3_historical, s3_alerts, s3 +from chalicelib import s3_historical, s3_alerts, s3 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" -WE_HAVE_ALERTS_SINCE = datetime.date(2017, 11, 6) +WE_HAVE_V2_ALERTS_SINCE = datetime.date(2017, 11, 6) +WE_HAVE_V3_ALERTS_SINCE = datetime.date(2024, 4, 12) def bucket_by( @@ -57,46 +58,11 @@ def use_S3(date, bus=False): return archival or bus -def partition_S3_dates(start_date: str | date, end_date: str | date, bus=False): - """ - Partitions dates by what data source they should be fetched from. - S3 is used for archival data and for bus data. API is used for recent (within 90 days) subway data. - TODO: Add Gobble data to this partitioning. - """ - CUTOFF = datetime.date.today() - datetime.timedelta(days=90) - - s3_dates = None - api_dates = None - - if end_date < CUTOFF or bus: - s3_dates = (start_date, end_date) - elif CUTOFF <= start_date: - api_dates = (start_date, end_date) - else: - s3_dates = (start_date, CUTOFF - datetime.timedelta(days=1)) - api_dates = (CUTOFF, end_date) - - return (s3_dates, api_dates) - - -def headways(start_date: str | date, stops, end_date: str | date | None = None): +def headways(start_date: date, stops, end_date: date | None = None): if end_date is None: - if use_S3(start_date, is_bus(stops)): - return s3_historical.headways(stops, start_date, start_date) - else: - return process_mbta_headways(stops, start_date) + return s3_historical.headways(stops, start_date, start_date) - s3_interval, api_interval = partition_S3_dates(start_date, end_date, is_bus(stops)) - all_data = [] - if s3_interval: - start, end = s3_interval - all_data.extend(s3_historical.headways(stops, start, end)) - - if api_interval: - start, end = api_interval - all_data.extend(process_mbta_headways(stops, start, end)) - - return all_data + return s3_historical.headways(stops, start_date, end_date) # Transit days run 3:30am-3:30am local time @@ -109,144 +75,61 @@ def current_transit_day(): return today -def process_mbta_headways(stops, start_date: str | date, end_date: str | date | None = None): - # get data - api_data = MbtaPerformanceAPI.get_api_data("headways", {"stop": stops}, start_date, end_date) - # combine all headways data - headways = [] - for dict_data in api_data: - headways += dict_data.get("headways", []) - - # conversion - for headway_dict in headways: - # convert to datetime - headway_dict["current_dep_dt"] = stamp_to_dt(headway_dict.get("current_dep_dt")) - headway_dict["previous_dep_dt"] = stamp_to_dt(headway_dict.get("previous_dep_dt")) - # convert to int - headway_dict["benchmark_headway_time_sec"] = int(headway_dict.get("benchmark_headway_time_sec")) - headway_dict["headway_time_sec"] = int(headway_dict.get("headway_time_sec")) - headway_dict["direction"] = int(headway_dict.get("direction")) - - return sorted(headways, key=lambda x: x["current_dep_dt"]) - - -def travel_times(start_date, from_stops, to_stops, end_date: str | date | None = None): - if end_date is None: - if use_S3(start_date, is_bus(from_stops)): - return s3_historical.travel_times(from_stops, to_stops, start_date, start_date) - else: - return process_mbta_travel_times(from_stops, to_stops, start_date) - - s3_interval, api_interval = partition_S3_dates(start_date, end_date, is_bus(from_stops)) - all_data = [] - if s3_interval: - start, end = s3_interval - all_data.extend(s3_historical.travel_times(from_stops, to_stops, start, end)) - - if api_interval: - start, end = api_interval - all_data.extend(process_mbta_travel_times(from_stops, to_stops, start, end)) - return all_data - - -def process_mbta_travel_times(from_stops, to_stops, start_date: str | date, end_date: str | date | None = None): - # get data - api_data = MbtaPerformanceAPI.get_api_data( - "traveltimes", {"from_stop": from_stops, "to_stop": to_stops}, start_date, end_date - ) - # combine all travel times data, remove threshold flags from performance API, and dedupe on `dep_dt` - trips = {} - for dict_data in api_data: - for event in dict_data.get("travel_times", []): - dep_dt = event["dep_dt"] - if dep_dt not in trips: - trips[dep_dt] = { - "route_id": event["route_id"], - "direction": int(event["direction"]), - # convert to datetime - "dep_dt": stamp_to_dt(event["dep_dt"]), - "arr_dt": stamp_to_dt(event["arr_dt"]), - # convert to int - "travel_time_sec": int(event["travel_time_sec"]), - "benchmark_travel_time_sec": int(event["benchmark_travel_time_sec"]), - } - trips_list = list(trips.values()) - return sorted(trips_list, key=lambda x: x["dep_dt"]) - - -def dwells(start_date, stops, end_date: str | date | None = None): +def travel_times(start_date: date, from_stops, to_stops, end_date: date | None = None): if end_date is None: - if use_S3(start_date, is_bus(stops)): - return s3_historical.dwells(stops, start_date, start_date) - else: - return process_mbta_dwells(stops, start_date) - - s3_interval, api_interval = partition_S3_dates(start_date, end_date, is_bus(stops)) - all_data = [] - if s3_interval: - start, end = s3_interval - all_data.extend(s3_historical.dwells(stops, start, end)) - - if api_interval: - start, end = api_interval - all_data.extend(process_mbta_dwells(stops, start, end)) + return s3_historical.travel_times(from_stops, to_stops, start_date, start_date) - return all_data + return s3_historical.travel_times(from_stops, to_stops, start_date, end_date) -def process_mbta_dwells(stops, start_date: str | date, end_date: str | date | None = None): - # get data - api_data = MbtaPerformanceAPI.get_api_data("dwells", {"stop": stops}, start_date, end_date) - - # combine all travel times data - dwells = [] - for dict_data in api_data: - dwells += dict_data.get("dwell_times", []) - - # conversion - for dwell_dict in dwells: - # convert to datetime - dwell_dict["arr_dt"] = stamp_to_dt(dwell_dict.get("arr_dt")) - dwell_dict["dep_dt"] = stamp_to_dt(dwell_dict.get("dep_dt")) - # convert to int - dwell_dict["dwell_time_sec"] = int(dwell_dict.get("dwell_time_sec")) - dwell_dict["direction"] = int(dwell_dict.get("direction")) +def dwells(start_date, stops, end_date: date | None = None): + if end_date is None: + return s3_historical.dwells(stops, start_date, start_date) - return sorted(dwells, key=lambda x: x["arr_dt"]) + return s3_historical.dwells(stops, start_date, end_date) -def alerts(day, params): +def alerts(day: date, params): try: - # Grab the current "transit day" (3:30am-3:30am) - today = current_transit_day() - # yesterday + 1 bonus day to cover the gap, since aws is only populated at 5/6am. - yesterday = today - datetime.timedelta(days=2) - # Use the API for today and yesterday's transit day, otherwise us. - if day >= yesterday: - api_data = MbtaPerformanceAPI.get_api_data("pastalerts", params, day) - elif day >= WE_HAVE_ALERTS_SINCE: + if day >= WE_HAVE_V2_ALERTS_SINCE and day <= WE_HAVE_V3_ALERTS_SINCE: # This is stupid because we're emulating MBTA-performance ick - api_data = [{"past_alerts": s3_alerts.get_alerts(day, params["route"])}] + alert_items = s3_alerts.get_v2_alerts(day, params["route"]) + elif day >= WE_HAVE_V3_ALERTS_SINCE: + # fetch s3 v3 data + alert_items = s3_alerts.get_v3_alerts(day, params["route"]) else: return None - # combine all alerts data - alert_items = [] - for dict_data in api_data: - alert_items = alert_items + dict_data.get("past_alerts", []) - # get data flat_alerts = [] for alert_item in alert_items: - for alert_version in alert_item["alert_versions"]: - flat_alerts.append( - { - "valid_from": stamp_to_dt(int(alert_version["valid_from"])), - "valid_to": stamp_to_dt(int(alert_version["valid_to"])), - "text": alert_version["header_text"], - } - ) + if "alert_versions" in alert_item: + try: + for alert_version in alert_item["alert_versions"]: + flat_alerts.append( + { + "valid_from": stamp_to_dt(int(alert_version["valid_from"])), + "valid_to": stamp_to_dt(int(alert_version["valid_to"])), + "text": alert_version["header_text"], + } + ) + except KeyError as e: + print(f"Handled KeyError: Couldn't access {e} from alert {alert_item}") + elif "attributes" in alert_item and any( + alert_item["attributes"]["effect"] == x for x in ["DETOUR", "DELAY", "STOP_CLOSURE", "SERVICE_CHANGE"] + ): + try: + for alert_version in alert_item["attributes"]["active_period"]: + flat_alerts.append( + { + "valid_from": alert_version["start"], + "valid_to": alert_version["end"], + "text": alert_item["attributes"]["short_header"] or alert_item["attributes"]["header"], + } + ) + except KeyError as e: + print(f"Handled KeyError: Couldn't access {e} from alert {alert_item}") return flat_alerts except Exception: traceback.print_exc() diff --git a/server/chalicelib/date_utils.py b/server/chalicelib/date_utils.py index 21f5ac5a..dd71e3a2 100644 --- a/server/chalicelib/date_utils.py +++ b/server/chalicelib/date_utils.py @@ -11,6 +11,13 @@ MAX_MONTH_DATA_DATE = "2024-03-31" +def get_max_monthly_data_date(): + """ + Returns the most recent date for which we have monthly data + """ + return datetime.strptime(MAX_MONTH_DATA_DATE, "%Y-%m-%d").date() + + def parse_event_date(date_str: str): if len(date_str) == 19: return datetime.strptime(date_str, DATE_FORMAT_MASSDOT).replace(tzinfo=EASTERN_TIME) diff --git a/server/chalicelib/parallel.py b/server/chalicelib/parallel.py index cffe4352..692a676b 100644 --- a/server/chalicelib/parallel.py +++ b/server/chalicelib/parallel.py @@ -1,7 +1,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import date import pandas as pd -from chalicelib.date_utils import MAX_MONTH_DATA_DATE +from chalicelib import date_utils def make_parallel(single_func, THREAD_COUNT=5): @@ -26,22 +27,25 @@ def date_range(start: str, end: str): return pd.date_range(start, end) -def s3_date_range(start: str, end: str): +def s3_date_range(start: date, end: date): """ Generates a date range, meant for s3 data For all dates that we have monthly datasets for, return 1 date of the month For all dates that we have daily datasets for, return all dates """ month_end = end - if pd.to_datetime(MAX_MONTH_DATA_DATE) < pd.to_datetime(end): - month_end = MAX_MONTH_DATA_DATE + if date_utils.get_max_monthly_data_date() < end and date_utils.get_max_monthly_data_date() > start: + month_end = date_utils.get_max_monthly_data_date() + + date_range = pd.date_range(start, month_end, freq="1D", inclusive="both") # This is kinda funky, but is stil simpler than other approaches # pandas won't generate a monthly date_range that includes Jan and Feb for Jan31-Feb1 e.g. # So we generate a daily date_range and then resample it down (summing 0s as a no-op in the process) so it aligns. - dates = pd.date_range(start, month_end, freq="1D", inclusive="both") - series = pd.Series(0, index=dates) - months = series.resample("1M").sum().index + if date_utils.get_max_monthly_data_date() > start: + dates = pd.date_range(start, month_end, freq="1D", inclusive="both") + series = pd.Series(0, index=dates) + date_range = series.resample("1M").sum().index # all dates between month_end and end if month_end is less than end if pd.to_datetime(month_end) < pd.to_datetime(end): @@ -49,6 +53,6 @@ def s3_date_range(start: str, end: str): # combine the two date ranges of months and dates if daily_dates is not None and len(daily_dates) > 0: - months = months.union(daily_dates) + date_range = date_range.union(daily_dates) - return months + return date_range diff --git a/server/chalicelib/s3.py b/server/chalicelib/s3.py index 360905ce..30510b63 100644 --- a/server/chalicelib/s3.py +++ b/server/chalicelib/s3.py @@ -1,12 +1,14 @@ from datetime import date import boto3 import botocore +import pandas as pd from botocore.exceptions import ClientError import csv import itertools import zlib from chalicelib import parallel +from chalicelib import date_utils BUCKET = "tm-mbta-performance" s3 = boto3.client("s3", config=botocore.client.Config(max_pool_connections=15)) @@ -33,34 +35,44 @@ def is_bus(stop_id: str): return ("-0-" in stop_id) or ("-1-" in stop_id) -def get_live_folder(stop_id: str): +def get_gobble_folder(stop_id: str): if is_bus(stop_id): return "daily-bus-data" else: return "daily-rapid-data" -def download_one_event_file(date, stop_id: str, use_live_data=False): +def get_lamp_folder(): + return "daily-data" + + +def download_one_event_file(date: pd.Timestamp, stop_id: str, use_gobble=False): """As advertised: single event file from s3""" year, month, day = date.year, date.month, date.day - if use_live_data: - folder = get_live_folder(stop_id) - key = f"Events-live/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" + # if current date is newer than the max monthly data date, use LAMP + if date.date() > date_utils.get_max_monthly_data_date(): + # if we've asked to use gobble data or bus data, check gobble + if use_gobble or is_bus(stop_id): + folder = get_gobble_folder(stop_id) + key = f"Events-live/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz" + else: + folder = get_lamp_folder() + key = f"Events-lamp/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv" else: folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data" key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz" # Download events from S3 try: - decompressed = download(key, "ascii") + decompressed = download(key, "ascii", ".gz" in key) except ClientError as ex: if ex.response["Error"]["Code"] == "NoSuchKey": # raise Exception(f"Data not available on S3 for key {key} ") from None print(f"WARNING: No data available on S3 for key: {key}") - if not use_live_data and is_bus(stop_id): - return download_one_event_file(date, stop_id, use_live_data=True) + if not use_gobble and not is_bus(stop_id): + return download_one_event_file(date, stop_id, use_gobble=True) return [] else: raise @@ -76,13 +88,12 @@ def download_one_event_file(date, stop_id: str, use_live_data=False): @parallel.make_parallel -def parallel_download_events(datestop): +def parallel_download_events(datestop: itertools.product): (date, stop) = datestop - # TODO: Force gobble when date is past the max monthly data date return download_one_event_file(date, stop) -def download_events(start_date: str | date, end_date: str | date, stops: list): +def download_events(start_date: date, end_date: date, stops: list): datestops = itertools.product(parallel.s3_date_range(start_date, end_date), stops) result = parallel_download_events(datestops) result = filter( diff --git a/server/chalicelib/s3_alerts.py b/server/chalicelib/s3_alerts.py index 680265a4..7c4dbf8d 100644 --- a/server/chalicelib/s3_alerts.py +++ b/server/chalicelib/s3_alerts.py @@ -1,25 +1,37 @@ +from datetime import date import json -from chalicelib import MbtaPerformanceAPI, s3 +from chalicelib import s3 def routes_for_alert(alert): routes = set() - try: - for alert_version in alert["alert_versions"]: - for informed_entity in alert_version["informed_entity"]: - if "route_id" in informed_entity: - routes.add(informed_entity["route_id"]) - except KeyError as e: - print(f"Handled KeyError: Couldn't access {e} from alert {alert}") + + if "alert_versions" in alert: + try: + for alert_version in alert["alert_versions"]: + for informed_entity in alert_version["informed_entity"]: + if "route_id" in informed_entity: + routes.add(informed_entity["route_id"]) + except KeyError as e: + print(f"Handled KeyError: Couldn't access {e} from alert {alert}") + elif "attributes" in alert: + try: + for informed_entity in alert["attributes"]["informed_entity"]: + if "route" in informed_entity: + routes.add(informed_entity["route"]) + except KeyError as e: + print(f"Handled KeyError: Couldn't access {e} from alert {alert}") return routes -def key(day): +def key(day, v3: bool = False): + if v3: + return f"Alerts/v3/{str(day)}.json.gz" return f"Alerts/{str(day)}.json.gz" -def get_alerts(day, routes): +def get_v2_alerts(day: date, routes): alerts_str = s3.download(key(day), "utf8") alerts = json.loads(alerts_str)[0]["past_alerts"] @@ -30,7 +42,12 @@ def matches_route(alert): return list(filter(matches_route, alerts)) -def store_alerts(day): - api_data = MbtaPerformanceAPI.get_api_data("pastalerts", {}, day) - alerts = json.dumps(api_data).encode("utf8") - s3.upload(key(day), alerts, True) +def get_v3_alerts(day: date, routes: list[str]): + alerts_str = s3.download(key(day, v3=True), "utf8") + alerts = json.loads(alerts_str) + + def matches_route(alert): + targets = routes_for_alert(alert) + return any(r in targets for r in routes) + + return list(filter(matches_route, alerts.values())) diff --git a/server/chalicelib/s3_historical.py b/server/chalicelib/s3_historical.py index 55dcf8bf..47078106 100644 --- a/server/chalicelib/s3_historical.py +++ b/server/chalicelib/s3_historical.py @@ -33,7 +33,7 @@ def unique_everseen(iterable, key=None): yield element -def dwells(stop_ids: list, start_date: str | date, end_date: str | date): +def dwells(stop_ids: list, start_date: date, end_date: date): rows_by_time = s3.download_events(start_date, end_date, stop_ids) dwells = [] @@ -60,7 +60,7 @@ def dwells(stop_ids: list, start_date: str | date, end_date: str | date): return dwells -def headways(stop_ids: list, start_date: str | date, end_date: str | date): +def headways(stop_ids: list, start_date: date, end_date: date): rows_by_time = s3.download_events(start_date, end_date, stop_ids) only_departures = filter(lambda row: row["event_type"] in EVENT_DEPARTURE, rows_by_time) @@ -99,7 +99,7 @@ def headways(stop_ids: list, start_date: str | date, end_date: str | date): return headways -def travel_times(stops_a: list, stops_b: list, start_date: str | date, end_date: str | date): +def travel_times(stops_a: list, stops_b: list, start_date: date, end_date: date): rows_by_time_a = s3.download_events(start_date, end_date, stops_a) rows_by_time_b = s3.download_events(start_date, end_date, stops_b) @@ -130,11 +130,13 @@ def travel_times(stops_a: list, stops_b: list, start_date: str | date, end_date: # benchmark calculation: # not every file will have the scheduled_tt field, so we use get. - sched_arr = arrival.get("scheduled_tt") - sched_dep = departure.get("scheduled_tt") + sched_arr = arrival.get("scheduled_tt") or arrival.get("scheduled_travel_time") + sched_dep = departure.get("scheduled_tt") or departure.get("scheduled_travel_time") try: # sched values may be None or '' benchmark = float(sched_arr) - float(sched_dep) + if benchmark < 1: + benchmark = None except (TypeError, ValueError): benchmark = None diff --git a/server/cloudformation.json b/server/cloudformation.json index fa509581..80c2862f 100644 --- a/server/cloudformation.json +++ b/server/cloudformation.json @@ -32,9 +32,9 @@ "Type": "String", "Description": "The ACM ARN of the backend certificate." }, - "MbtaV2ApiKey": { + "MbtaV3ApiKey": { "Type": "String", - "Description": "MBTA-performance API key." + "Description": "MBTA V3 API key." }, "DDApiKey": { "Type": "String", @@ -92,7 +92,7 @@ "Handler": "datadog_lambda.handler.handler", "Environment": { "Variables": { - "MBTA_V2_API_KEY": { "Ref": "MbtaV2ApiKey" }, + "MBTA_V3_API_KEY": { "Ref": "MbtaV3ApiKey" }, "DD_API_KEY": { "Ref": "DDApiKey" }, "DD_VERSION": { "Ref": "GitVersion" }, "DD_TAGS": { "Ref": "DDTags" } diff --git a/server/refill_alerts.py b/server/refill_alerts.py deleted file mode 100644 index 792c1519..00000000 --- a/server/refill_alerts.py +++ /dev/null @@ -1,38 +0,0 @@ -import datetime - -from chalicelib.parallel import date_range -from chalicelib.s3_alerts import store_alerts, get_alerts - -import sys - -if len(sys.argv) < 3: - print("usage: poetry run python refill_alerts.py 2021-04-12 2021-05-01") -start_str = sys.argv[1] -end_str = sys.argv[2] - -START = datetime.datetime.strptime(start_str, "%Y-%m-%d").date() -END = datetime.datetime.strptime(end_str, "%Y-%m-%d").date() - - -def do_alerts_exist(d): - try: - get_alerts(d, ["Red"]) - except Exception as err: - if err.response["Error"]["Code"] == "NoSuchKey": - return False - return True - - -for d in date_range(START, END): - d = d.date() - if do_alerts_exist(d): - continue - print("storing", d) - for i in range(3): - print(" attempt", i + 1, "of 3") - try: - store_alerts(d) - print("success") - break - except Exception: - continue