Merge pull request #254 from ibi-group/cdp-daily

CDP Daily entity upload to weekly folder

README.md

@@ -257,10 +257,14 @@ The special E2E client settings should be defined in `env.yml`:
 | BUGSNAG_EVENT_REQUEST_JOB_DELAY_IN_HOURS | integer | Optional | 24 | Frequency in hours to trigger event requests. |
 | BUGSNAG_PROJECT_NOTIFIER_API_KEY | string | Optional | 123e4567e89b12d3a4564266 | A valid Bugsnag project API key. |
 | BUGSNAG_REPORTING_WINDOW_IN_DAYS | integer | Optional | 14 | Specifies how far in the past events should be retrieved. |
+| BUGSNAG_WEBHOOK_PERMITTED_IPS | string | Optional | 104.196.245.109, 104.196.254.247 | Bugsnag IP addresses which webhook requests are expected to come from. |
+| CONNECTED_DATA_PLATFORM_REPORTING_INTERVAL | string | Optional | daily, hourly | Specifies how often to aggregate and upload usage data. Defaults to hourly. |
+| CONNECTED_DATA_PLATFORM_FOLDER_GROUPING | string | Optional | none, weekly-monday-sunday | Specifies how to organize uploaded files. 'none' puts all uploaded files to the same folder CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME. 'weekly-monday-sunday' creates a folder of the form `<CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME>_<YYYY-MM-DD>_<YYYY-MM-DD>` where the start date is a Monday and the end date is the following Sunday. Defaults to none. |
 | CONNECTED_DATA_PLATFORM_S3_BUCKET_NAME | string | Optional | bucket-name | Specifies the S3 bucket name for the CDP trip history push. |
 | CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME | string | Optional | folder-name | Specifies the S3 folder name for the CDP trip history push. |
+| CONNECTED_DATA_PLATFORM_REPORTED_ENTITIES | object | Optional | { "MonitoredTrip": "all", "OtpUser": "all", "TripRequest": "interval" } | Use 'all' to report all full records. Use 'interval' to report full records whose dateCreated is within a reporting interval (e.g. day, hour). For TripRequest, you can use 'all anonymized' or 'interval anonymized' to anonymize records. Omitted entities are ignored. |
 | CONNECTED_DATA_PLATFORM_TRIP_HISTORY_UPLOAD_JOB_FREQUENCY_IN_MINUTES | integer | Optional | 5 | CDP trip history upload frequency. |
-| BUGSNAG_WEBHOOK_PERMITTED_IPS | string | Optional | 104.196.245.109, 104.196.254.247 | Bugsnag IP addresses which webhook requests are expected to come from. |
+| CONNECTED_DATA_PLATFORM_UPLOAD_BLANK_FILES | boolean | Optional | true | Whether to upload files where no records have been written. Defaults to true. |
 | DEFAULT_USAGE_PLAN_ID | string | Required | 123e45 | AWS API gateway default usage plan used when creating API keys for API users. |
 | MAXIMUM_PERMITTED_MONITORED_TRIPS | integer | Optional | 5 | The maximum number of saved monitored trips. |
 | MONGO_DB_NAME | string | Required | otp_middleware | The name of the OTP Middleware Mongo DB. |

configurations/default/env.yml.tmp

@@ -14,7 +14,16 @@ BUGSNAG_WEBHOOK_PERMITTED_IPS: 104.196.245.109, 104.196.254.247
 CONNECTED_DATA_PLATFORM_ENABLED: true
 CONNECTED_DATA_PLATFORM_S3_BUCKET_NAME: bucket-name
 CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME: folder-name
+CONNECTED_DATA_PLATFORM_REPORTING_INTERVAL: hourly
+CONNECTED_DATA_PLATFORM_FOLDER_GROUPING: none
 CONNECTED_DATA_PLATFORM_TRIP_HISTORY_UPLOAD_JOB_FREQUENCY_IN_MINUTES: 5
+CONNECTED_DATA_PLATFORM_UPLOAD_BLANK_FILES: true
+CONNECTED_DATA_PLATFORM_REPORTED_ENTITIES:
+  MonitoredTrip: all
+  OtpUser: all
+  TrackedJourney: interval
+  TripRequest: interval
+  TripSummary: interval
 
 AWS_PROFILE: default
 # AWS_API_SERVER and AWS_API_STAGE are for generating the swagger document at runtime.

src/main/java/org/opentripplanner/middleware/connecteddataplatform/FolderGrouping.java

@@ -0,0 +1,11 @@
+package org.opentripplanner.middleware.connecteddataplatform;
+
+/**
+ * Determines how folders should be organized.
+ */
+public enum FolderGrouping {
+    /** No grouping is taking place. */
+    NONE,
+    /** Folders are organized weekly, from Monday to Sunday. */
+    WEEKLY_MONDAY_SUNDAY
+}

src/main/java/org/opentripplanner/middleware/connecteddataplatform/ReportedEntities.java

@@ -0,0 +1,43 @@
+package org.opentripplanner.middleware.connecteddataplatform;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.opentripplanner.middleware.persistence.Persistence;
+import org.opentripplanner.middleware.persistence.TypedPersistence;
+import org.opentripplanner.middleware.utils.ConfigUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Singleton that holds configuration data about entities to report.
+ * Note: Fields are PascalCased to reflect the casing of respective class names.
+ */
+public class ReportedEntities {
+    public static final Map<String, TypedPersistence<?>> persistenceMap = Map.of(
+    "MonitoredTrip", Persistence.monitoredTrips,
+    "OtpUser", Persistence.otpUsers,
+    "TrackedJourney", Persistence.trackedJourneys,
+    "TripRequest", Persistence.tripRequests,
+    "TripSummary", Persistence.tripSummaries
+    );
+
+    public static final Map<String, String> entityMap = Map.copyOf(
+        getEntityMap(ConfigUtils.getConfigProperty("CONNECTED_DATA_PLATFORM_REPORTED_ENTITIES"))
+    );
+
+    private ReportedEntities() {}
+
+    static Map<String, String> getEntityMap(JsonNode node) {
+        if (node == null || node.isEmpty()) return Map.of();
+
+        // Only include keys that are in persistenceMap.
+        Map<String, String> map = new HashMap<>();
+        for (String key : persistenceMap.keySet()) {
+            if (node.has(key)) {
+                map.put(key, node.get(key).textValue());
+            }
+        }
+
+        return map;
+    }
+}

src/main/java/org/opentripplanner/middleware/connecteddataplatform/ReportingInterval.java

@@ -0,0 +1,11 @@
+package org.opentripplanner.middleware.connecteddataplatform;
+
+/**
+ * Determines how often data should be reported.
+ */
+public enum ReportingInterval {
+    /** Data is reported hourly. */
+    HOURLY,
+    /** Data is reported daily. */
+    DAILY
+}

src/main/java/org/opentripplanner/middleware/connecteddataplatform/TripHistoryUploadJob.java

@@ -7,7 +7,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
+import java.util.Map;
 
 /**
  * This job is responsible for keeping the trip history held on s3 up-to-date by defining the hours which should be
@@ -19,8 +21,12 @@ public class TripHistoryUploadJob implements Runnable {
     private static final int HISTORIC_UPLOAD_HOURS_BACK_STOP = 24;
 
     public void run() {
-        stageUploadHours();
-        processTripHistory(false);
+        if (ConnectedDataManager.isReportingDaily()) {
+            stageUploadDays();
+        } else {
+            stageUploadHours();
+        }
+        processTripHistory(ConnectedDataManager.CONNECTED_DATA_PLATFORM_REPORTING_INTERVAL, null);
     }
 
     /**
@@ -29,31 +35,52 @@ public void run() {
      * if not already accounted for.
      */
     public static void stageUploadHours() {
-        LocalDateTime previousWholeHourFromNow = DateTimeUtils.getPreviousWholeHourFromNow();
+        stageUploadTimes(DateTimeUtils.getPreviousWholeHourFrom(LocalDateTime.now()), ChronoUnit.HOURS);
+    }
+
+    /**
+     * Add to the trip history upload list any days between the previous day and the last created (pending or
+     * completed) trip history upload. This will cover any days missed due to downtime and add the latest upload day
+     * if not already accounted for.
+     */
+    public static void stageUploadDays() {
+        stageUploadTimes(DateTimeUtils.getPreviousDayFrom(LocalDateTime.now()), ChronoUnit.DAYS);
+    }
+
+    /**
+     * Add to the trip history upload list any hours/days between the previous whole hour/day and the last created
+     * (pending or completed) trip history upload. This will cover any hours/days missed due to downtime,
+     * up to HISTORIC_UPLOAD_HOURS_BACK_STOP hours, and add the latest upload hour/day if not already accounted for.
+     */
+    private static void stageUploadTimes(LocalDateTime previousTime, ChronoUnit chronoUnit) {
         TripHistoryUpload lastCreated = TripHistoryUpload.getLastCreated();
         if (lastCreated == null) {
             // Stage first ever upload hour.
-            Persistence.tripHistoryUploads.create(new TripHistoryUpload(previousWholeHourFromNow));
-            LOG.debug("Staging first ever upload hour: {}.", previousWholeHourFromNow);
+            Persistence.tripHistoryUploads.create(new TripHistoryUpload(previousTime));
+            LOG.debug("Staging first ever upload hour: {}.", previousTime);
             return;
         }
-        // Stage all hours between the last hour uploaded and an hour ago.
-        List<LocalDateTime> betweenHours = DateTimeUtils.getHoursBetween(lastCreated.uploadHour, previousWholeHourFromNow);
-        betweenHours.forEach(uploadHour -> {
+        // Stage all time between the last time uploaded and an hour/day ago.
+        List<LocalDateTime> intermediateTimes = DateTimeUtils.getTimeUnitsBetween(
+            lastCreated.uploadHour,
+            previousTime,
+            chronoUnit
+        );
+        intermediateTimes.forEach(uploadHour -> {
             if (uploadHour.isAfter(getHistoricDateTimeBackStop())) {
                 LOG.debug(
                     "Staging hour: {} that is between last created: {} and the previous whole hour: {}",
                     lastCreated,
-                    previousWholeHourFromNow,
+                    previousTime,
                     uploadHour
                 );
                 Persistence.tripHistoryUploads.create(new TripHistoryUpload(uploadHour));
             }
         });
-        if (!lastCreated.uploadHour.isEqual(previousWholeHourFromNow)) {
+        if (!lastCreated.uploadHour.isEqual(previousTime)) {
             // Last created is not the latest upload hour, so stage an hour ago.
-            Persistence.tripHistoryUploads.create(new TripHistoryUpload(previousWholeHourFromNow));
-            LOG.debug("Last created {} is older than the latest {}, so staging.", lastCreated, previousWholeHourFromNow);
+            Persistence.tripHistoryUploads.create(new TripHistoryUpload(previousTime));
+            LOG.debug("Last created {} is older than the latest {}, so staging.", lastCreated, previousTime);
         }
     }
 
@@ -70,18 +97,21 @@ private static LocalDateTime getHistoricDateTimeBackStop() {
      * Process incomplete upload dates. This will be uploads which are flagged as 'pending'. If the upload date is
      * compiled and uploaded successfully, it is flagged as 'complete'.
      */
-    public static void processTripHistory(boolean isTest) {
+    public static void processTripHistory(ReportingInterval reportingInterval, Map<String, String> reportedEntities) {
         List<TripHistoryUpload> incompleteUploads = ConnectedDataManager.getIncompleteUploads();
         incompleteUploads.forEach(tripHistoryUpload -> {
-            int numTripRequestsUpload = ConnectedDataManager.compileAndUploadTripHistory(tripHistoryUpload.uploadHour, isTest);
-            if (numTripRequestsUpload != Integer.MIN_VALUE) {
+            int numRecordsToUpload = ConnectedDataManager.compileAndUploadTripHistory(
+                tripHistoryUpload.uploadHour,
+                reportingInterval,
+                reportedEntities
+            );
+            if (numRecordsToUpload != Integer.MIN_VALUE) {
                 // If successfully compiled and updated, update the status to 'completed' and record the number of trip
                 // requests uploaded (if any).
                 tripHistoryUpload.status = TripHistoryUploadStatus.COMPLETED.getValue();
-                tripHistoryUpload.numTripRequestsUploaded = numTripRequestsUpload;
+                tripHistoryUpload.numTripRequestsUploaded = numRecordsToUpload;
                 Persistence.tripHistoryUploads.replace(tripHistoryUpload.id, tripHistoryUpload);
             }
         });
     }
-
 }

src/main/java/org/opentripplanner/middleware/utils/DateTimeUtils.java

@@ -264,6 +264,13 @@ public static Date getEndOfHour(LocalDateTime date) {
         return convertToDate(date.truncatedTo(ChronoUnit.HOURS).plusHours(1).minusSeconds(1));
     }
 
+    /**
+     * Get the end of a day from a {@link LocalDateTime} object which is returned as a {@link Date} object.
+     */
+    public static Date getEndOfDay(LocalDateTime date) {
+        return convertToDate(date.truncatedTo(ChronoUnit.DAYS).plusDays(1).minusSeconds(1));
+    }
+
     /**
      * Get the start of the current hour.
      */
@@ -272,27 +279,51 @@ public static LocalDateTime getStartOfCurrentHour() {
     }
 
     /**
-     * Get the hours between two {@link LocalDateTime} objects. The list of {@link LocalDateTime} objects returned does
-     * not include the originally provided hours.
+     * Get the hours between two {@link LocalDateTime} objects, excluding the start and end hours.
      */
     public static List<LocalDateTime> getHoursBetween(LocalDateTime start, LocalDateTime end) {
+        return getTimeUnitsBetween(start, end, ChronoUnit.HOURS);
+    }
+
+    /**
+     * Get the days between two {@link LocalDateTime} objects, excluding start and end days.
+     */
+    public static List<LocalDateTime> getDaysBetween(LocalDateTime start, LocalDateTime end) {
+        return getTimeUnitsBetween(start, end, ChronoUnit.DAYS);
+    }
+
+    /**
+     * Get the time units (e.g. days, hours) between two {@link LocalDateTime} objects, excluding start and end.
+     */
+    public static List<LocalDateTime> getTimeUnitsBetween(
+        LocalDateTime start,
+        LocalDateTime end,
+        ChronoUnit chronoUnit
+    ) {
         if (start.isAfter(end) || start.isEqual(end)) {
             LOG.warn("Start date/time: {} is after/equal to end date/time: {}.", start, end);
             return Lists.newArrayList();
         }
-        // Bump the start by one hour, so it is not included in the returned list.
-        start = start.plusHours(1).truncatedTo(ChronoUnit.HOURS);
+        // Bump the start by one day, so it is not included in the returned list.
+        start = start.plus(1, chronoUnit).truncatedTo(chronoUnit);
         return Stream
-            .iterate(start, date -> date.plusHours(1))
-            .limit(ChronoUnit.HOURS.between(start, end))
+            .iterate(start, date -> date.plus(1, chronoUnit))
+            .limit(chronoUnit.between(start, end))
             .collect(Collectors.toList());
     }
 
     /**
-     * Return the previous whole hour from now. E.g. If the time is 07:30, return 06:00.
+     * Return the previous whole hour from a given date. E.g. If the time is 07:30, return 06:00.
+     */
+    public static LocalDateTime getPreviousWholeHourFrom(LocalDateTime dateTime) {
+        return dateTime.truncatedTo(ChronoUnit.HOURS).minusHours(1);
+    }
+
+    /**
+     * Return the previous day from now. E.g. If today is Wednesday 07:30, return Tuesday.
      */
-    public static LocalDateTime getPreviousWholeHourFromNow() {
-        return LocalDateTime.now().truncatedTo(ChronoUnit.HOURS).minusHours(1);
+    public static LocalDateTime getPreviousDayFrom(LocalDateTime dateTime) {
+        return dateTime.truncatedTo(ChronoUnit.DAYS).minusDays(1);
     }
 
     /**

src/main/resources/env.schema.json

@@ -59,6 +59,21 @@
       "examples": ["14"],
       "description": "Specifies how far in the past events should be retrieved."
     },
+    "BUGSNAG_WEBHOOK_PERMITTED_IPS": {
+      "type": "string",
+      "examples": ["104.196.245.109, 104.196.254.247"],
+      "description": "Bugsnag IP addresses which webhook requests are expected to come from."
+    },
+    "CONNECTED_DATA_PLATFORM_REPORTING_INTERVAL": {
+      "type": "string",
+      "examples": ["daily", "hourly"],
+      "description": "Specifies how often to aggregate and upload usage data. Defaults to hourly."
+    },
+    "CONNECTED_DATA_PLATFORM_FOLDER_GROUPING": {
+      "type": "string",
+      "examples": ["none", "weekly-monday-sunday"],
+      "description": "Specifies how to organize uploaded files. 'none' puts all uploaded files to the same folder CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME. 'weekly-monday-sunday' creates a folder of the form `<CONNECTED_DATA_PLATFORM_S3_FOLDER_NAME>_<YYYY-MM-DD>_<YYYY-MM-DD>` where the start date is a Monday and the end date is the following Sunday. Defaults to none."
+    },
     "CONNECTED_DATA_PLATFORM_S3_BUCKET_NAME": {
       "type": "string",
       "examples": ["bucket-name"],
@@ -69,17 +84,44 @@
       "examples": ["folder-name"],
       "description": "Specifies the S3 folder name for the CDP trip history push."
     },
+    "CONNECTED_DATA_PLATFORM_REPORTED_ENTITIES": {
+      "type": "object",
+      "examples": ["{ \"MonitoredTrip\": \"all\", \"OtpUser\": \"all\", \"TripRequest\": \"interval\" }"],
+      "description": "Use 'all' to report all full records. Use 'interval' to report full records whose dateCreated is within a reporting interval (e.g. day, hour). For TripRequest, you can use 'all anonymized' or 'interval anonymized' to anonymize records. Omitted entities are ignored.",
+      "properties": {
+        "MonitoredTrip": {
+          "type": "string",
+          "examples": ["all"]
+        },
+        "OtpUser": {
+          "type": "string",
+          "examples": ["all"]
+        },
+        "TrackedJourney": {
+          "type": "string",
+          "examples": ["interval"]
+        },
+        "TripRequest": {
+          "type": "string",
+          "examples": ["interval"]
+        },
+        "TripSummary": {
+          "type": "string",
+          "examples": ["interval"]
+        }
+      }
+    },
     "CONNECTED_DATA_PLATFORM_TRIP_HISTORY_UPLOAD_JOB_FREQUENCY_IN_MINUTES": {
       "type": "integer",
       "examples": [
         "5"
       ],
       "description": "CDP trip history upload frequency."
     },
-    "BUGSNAG_WEBHOOK_PERMITTED_IPS": {
-      "type": "string",
-      "examples": ["104.196.245.109, 104.196.254.247"],
-      "description": "Bugsnag IP addresses which webhook requests are expected to come from."
+    "CONNECTED_DATA_PLATFORM_UPLOAD_BLANK_FILES": {
+      "type": "boolean",
+      "examples": ["true"],
+      "description": "Whether to upload files where no records have been written. Defaults to true."
     },
     "DEFAULT_USAGE_PLAN_ID": {
       "type": "string",

src/test/java/org/opentripplanner/middleware/connecteddataplatform/ReportedEntitiesTest.java

@@ -0,0 +1,27 @@
+package org.opentripplanner.middleware.connecteddataplatform;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.junit.jupiter.api.Test;
+import org.opentripplanner.middleware.testutils.OtpMiddlewareTestEnvironment;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class ReportedEntitiesTest extends OtpMiddlewareTestEnvironment {
+    @Test
+    void canGetEntityMap() {
+        ObjectNode node = JsonNodeFactory.instance.objectNode();
+        node.put("MonitoredTrip", "all");
+        node.put("TripRequest", "interval");
+        node.put("UnknownClass", "abc123");
+
+        Map<String, String> expected = Map.of(
+            "MonitoredTrip", "all",
+            "TripRequest", "interval"
+        );
+
+        assertEquals(expected, ReportedEntities.getEntityMap(node));
+    }
+}

src/test/java/org/opentripplanner/middleware/utils/DateTimeUtilsTest.java

@@ -1,16 +1,24 @@
 package org.opentripplanner.middleware.utils;
 
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import java.time.LocalDateTime;
 import java.time.ZonedDateTime;
 import java.util.Date;
+import java.util.List;
 import java.util.Locale;
 import java.util.stream.Stream;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.text.MatchesPattern.matchesPattern;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.opentripplanner.middleware.utils.DateTimeUtils.getDaysBetween;
+import static org.opentripplanner.middleware.utils.DateTimeUtils.getHoursBetween;
+import static org.opentripplanner.middleware.utils.DateTimeUtils.getPreviousDayFrom;
+import static org.opentripplanner.middleware.utils.DateTimeUtils.getPreviousWholeHourFrom;
 
 class DateTimeUtilsTest {
     @ParameterizedTest
@@ -37,4 +45,46 @@ private static Stream<Arguments> createDateFormatCases() {
             Arguments.of("tl", "5:44[\\u202f ]PM")
         );
     }
+
+    @Test
+    void canGetPreviousDay() {
+        var date = LocalDateTime.of(2024, 8, 10, 15, 34, 17);
+        var expectedDate = LocalDateTime.of(2024, 8, 9, 0, 0, 0);
+        assertEquals(expectedDate, getPreviousDayFrom(date));
+    }
+
+    @Test
+    void canGetPreviousWholeHour() {
+        var date = LocalDateTime.of(2024, 8, 10, 15, 34, 17);
+        var expectedDate = LocalDateTime.of(2024, 8, 10, 14, 0, 0);
+        assertEquals(expectedDate, getPreviousWholeHourFrom(date));
+    }
+
+    @Test
+    void canGetDaysBetween() {
+        var date1 = LocalDateTime.of(2024, 8, 10, 15, 34, 17);
+        var date2 = LocalDateTime.of(2024, 8, 15, 9, 55, 32);
+        var expectedDays = List.of(
+            LocalDateTime.of(2024, 8, 11, 0, 0, 0),
+            LocalDateTime.of(2024, 8, 12, 0, 0, 0),
+            LocalDateTime.of(2024, 8, 13, 0, 0, 0),
+            LocalDateTime.of(2024, 8, 14, 0, 0, 0)
+        );
+        assertEquals(expectedDays, getDaysBetween(date1, date2));
+    }
+
+    @Test
+    void canGetHoursBetween() {
+        var date1 = LocalDateTime.of(2024, 8, 10, 20, 34, 17);
+        var date2 = LocalDateTime.of(2024, 8, 11, 3, 55, 32);
+        var expectedHours = List.of(
+            LocalDateTime.of(2024, 8, 10, 21, 0, 0),
+            LocalDateTime.of(2024, 8, 10, 22, 0, 0),
+            LocalDateTime.of(2024, 8, 10, 23, 0, 0),
+            LocalDateTime.of(2024, 8, 11, 0, 0, 0),
+            LocalDateTime.of(2024, 8, 11, 1, 0, 0),
+            LocalDateTime.of(2024, 8, 11, 2, 0, 0)
+        );
+        assertEquals(expectedHours, getHoursBetween(date1, date2));
+    }
 }

src/test/resources/snapshots/org/opentripplanner/middleware/ConnectedDataPlatform/ConnectedDataPlatformTest/canCreateZipFileForTripSummaryWithError-0.json

@@ -1 +1 @@
-"[{\"requestId\":\"783726\",\"fromPlace\":null,\"toPlace\":null,\"date\":\" 2021-09-22\",\"time\":\"15:54\",\"timeSelection\":\"DEPART_AT\",\"mode\":[\"WALK\",\"BUS\",\"RAIL\"],\"maxWalkDistance\":\"1027\",\"optimize\":\"QUICK\",\"itineraries\":[],\"error\":{\"id\":400,\"msg\":\"Trip is not possible.  You might be trying to plan a trip outside the map data boundary.\",\"missing\":[\"from\"],\"noPath\":true}}]"
+"[{\"requestId\":\"783726\",\"fromPlace\":null,\"toPlace\":null,\"date\":\"2021-09-22\",\"time\":\"15:54\",\"timeSelection\":\"DEPART_AT\",\"mode\":[\"WALK\",\"BUS\",\"RAIL\"],\"maxWalkDistance\":null,\"optimize\":null,\"itineraries\":[],\"error\":{\"id\":400,\"msg\":\"Trip is not possible.  You might be trying to plan a trip outside the map data boundary.\",\"missing\":[\"from\"],\"noPath\":true}}]"

src/test/resources/snapshots/org/opentripplanner/middleware/ConnectedDataPlatform/ConnectedDataPlatformTest/canCreateZipFileWithContent-0.json

@@ -1 +1 @@
-"[{\"requestId\":\"783726\",\"fromPlace\":null,\"toPlace\":null,\"date\":\" 2021-09-22\",\"time\":\"15:54\",\"timeSelection\":\"DEPART_AT\",\"mode\":[\"WALK\",\"BUS\",\"RAIL\"],\"maxWalkDistance\":\"1027\",\"optimize\":\"QUICK\",\"itineraries\":[{\"tripSummaryId\":1,\"itineraryIndex\":1,\"duration\":1114,\"startTime\":1591717210000,\"endTime\":1591718324000,\"transfers\":0,\"transitTime\":240,\"waitingTime\":2,\"walkDistance\":1256.0514029764959,\"walkTime\":872,\"legs\":[{\"distance\":837.5169999999998,\"duration\":584.0,\"startTime\":1591717210000,\"endTime\":1591717794000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:9758\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":893.8869820964361,\"duration\":240.0,\"startTime\":1591717795000,\"endTime\":1591718035000,\"mode\":\"TRAM\",\"transitLeg\":true,\"fromStop\":\"TriMet:9758\",\"from\":{\"lon\":-122.689886,\"lat\":45.521321},\"toStop\":\"TriMet:8334\",\"to\":{\"lon\":-122.679145,\"lat\":45.518496},\"agencyId\":\"TRIMET\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:100\",\"routeShortName\":null,\"routeLongName\":\"MAX Blue Line\",\"routeType\":0,\"tripBlockId\":\"9004\",\"tripId\":\"TriMet:9942889\",\"rentedVehicle\":null},{\"distance\":418.19,\"duration\":288.0,\"startTime\":1591718036000,\"endTime\":1591718324000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:8334\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]},{\"tripSummaryId\":1,\"itineraryIndex\":2,\"duration\":1242,\"startTime\":1591717148000,\"endTime\":1591718390000,\"transfers\":0,\"transitTime\":420,\"waitingTime\":2,\"walkDistance\":1297.4104029783646,\"walkTime\":820,\"legs\":[{\"distance\":578.6750000000001,\"duration\":351.0,\"startTime\":1591717148000,\"endTime\":1591717499000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:10753\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":1103.6887432320718,\"duration\":420.0,\"startTime\":1591717500000,\"endTime\":1591717920000,\"mode\":\"TRAM\",\"transitLeg\":true,\"fromStop\":\"TriMet:10753\",\"from\":{\"lon\":-122.682374,\"lat\":45.528742},\"toStop\":\"TriMet:9633\",\"to\":{\"lon\":-122.683873,\"lat\":45.519059},\"agencyId\":\"PSC\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:195\",\"routeShortName\":null,\"routeLongName\":\"Portland Streetcar - B Loop\",\"routeType\":0,\"tripBlockId\":\"58\",\"tripId\":\"TriMet:9945177\",\"rentedVehicle\":null},{\"distance\":718.4660000000001,\"duration\":469.0,\"startTime\":1591717921000,\"endTime\":1591718390000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:9633\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]},{\"tripSummaryId\":1,\"itineraryIndex\":3,\"duration\":1036,\"startTime\":1591717825000,\"endTime\":1591718861000,\"transfers\":0,\"transitTime\":360,\"waitingTime\":2,\"walkDistance\":966.7338656666136,\"walkTime\":674,\"legs\":[{\"distance\":802.1549999999999,\"duration\":574.0,\"startTime\":1591717825000,\"endTime\":1591718399000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:6911\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":1286.9349137891197,\"duration\":360.0,\"startTime\":1591718400000,\"endTime\":1591718760000,\"mode\":\"BUS\",\"transitLeg\":true,\"fromStop\":\"TriMet:6911\",\"from\":{\"lon\":-122.690453,\"lat\":45.52188},\"toStop\":\"TriMet:5020\",\"to\":{\"lon\":-122.678854,\"lat\":45.516794},\"agencyId\":\"TRIMET\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:15\",\"routeShortName\":\"15\",\"routeLongName\":\"Belmont/NW 23rd\",\"routeType\":3,\"tripBlockId\":\"1505\",\"tripId\":\"TriMet:9932776\",\"rentedVehicle\":null},{\"distance\":164.377,\"duration\":100.0,\"startTime\":1591718761000,\"endTime\":1591718861000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:5020\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]}],\"error\":null}]"
+"[{\"requestId\":\"783726\",\"fromPlace\":null,\"toPlace\":null,\"date\":\"2021-09-22\",\"time\":\"15:54\",\"timeSelection\":\"DEPART_AT\",\"mode\":[\"WALK\",\"BUS\",\"RAIL\"],\"maxWalkDistance\":null,\"optimize\":null,\"itineraries\":[{\"tripSummaryId\":1,\"itineraryIndex\":1,\"duration\":1114,\"startTime\":1591717210000,\"endTime\":1591718324000,\"transfers\":0,\"transitTime\":240,\"waitingTime\":2,\"walkDistance\":1256.0514029764959,\"walkTime\":872,\"legs\":[{\"distance\":837.5169999999998,\"duration\":584.0,\"startTime\":1591717210000,\"endTime\":1591717794000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:9758\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":893.8869820964361,\"duration\":240.0,\"startTime\":1591717795000,\"endTime\":1591718035000,\"mode\":\"TRAM\",\"transitLeg\":true,\"fromStop\":\"TriMet:9758\",\"from\":{\"lon\":-122.689886,\"lat\":45.521321},\"toStop\":\"TriMet:8334\",\"to\":{\"lon\":-122.679145,\"lat\":45.518496},\"agencyId\":\"TRIMET\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:100\",\"routeShortName\":null,\"routeLongName\":\"MAX Blue Line\",\"routeType\":0,\"tripBlockId\":\"9004\",\"tripId\":\"TriMet:9942889\",\"rentedVehicle\":null},{\"distance\":418.19,\"duration\":288.0,\"startTime\":1591718036000,\"endTime\":1591718324000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:8334\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]},{\"tripSummaryId\":1,\"itineraryIndex\":2,\"duration\":1242,\"startTime\":1591717148000,\"endTime\":1591718390000,\"transfers\":0,\"transitTime\":420,\"waitingTime\":2,\"walkDistance\":1297.4104029783646,\"walkTime\":820,\"legs\":[{\"distance\":578.6750000000001,\"duration\":351.0,\"startTime\":1591717148000,\"endTime\":1591717499000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:10753\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":1103.6887432320718,\"duration\":420.0,\"startTime\":1591717500000,\"endTime\":1591717920000,\"mode\":\"TRAM\",\"transitLeg\":true,\"fromStop\":\"TriMet:10753\",\"from\":{\"lon\":-122.682374,\"lat\":45.528742},\"toStop\":\"TriMet:9633\",\"to\":{\"lon\":-122.683873,\"lat\":45.519059},\"agencyId\":\"PSC\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:195\",\"routeShortName\":null,\"routeLongName\":\"Portland Streetcar - B Loop\",\"routeType\":0,\"tripBlockId\":\"58\",\"tripId\":\"TriMet:9945177\",\"rentedVehicle\":null},{\"distance\":718.4660000000001,\"duration\":469.0,\"startTime\":1591717921000,\"endTime\":1591718390000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:9633\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]},{\"tripSummaryId\":1,\"itineraryIndex\":3,\"duration\":1036,\"startTime\":1591717825000,\"endTime\":1591718861000,\"transfers\":0,\"transitTime\":360,\"waitingTime\":2,\"walkDistance\":966.7338656666136,\"walkTime\":674,\"legs\":[{\"distance\":802.1549999999999,\"duration\":574.0,\"startTime\":1591717825000,\"endTime\":1591718399000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":null,\"from\":null,\"toStop\":\"TriMet:6911\",\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false},{\"distance\":1286.9349137891197,\"duration\":360.0,\"startTime\":1591718400000,\"endTime\":1591718760000,\"mode\":\"BUS\",\"transitLeg\":true,\"fromStop\":\"TriMet:6911\",\"from\":{\"lon\":-122.690453,\"lat\":45.52188},\"toStop\":\"TriMet:5020\",\"to\":{\"lon\":-122.678854,\"lat\":45.516794},\"agencyId\":\"TRIMET\",\"interlineWithPreviousLeg\":false,\"realTime\":false,\"routeId\":\"TriMet:15\",\"routeShortName\":\"15\",\"routeLongName\":\"Belmont/NW 23rd\",\"routeType\":3,\"tripBlockId\":\"1505\",\"tripId\":\"TriMet:9932776\",\"rentedVehicle\":null},{\"distance\":164.377,\"duration\":100.0,\"startTime\":1591718761000,\"endTime\":1591718861000,\"mode\":\"WALK\",\"transitLeg\":false,\"fromStop\":\"TriMet:5020\",\"from\":null,\"toStop\":null,\"to\":null,\"agencyId\":null,\"interlineWithPreviousLeg\":null,\"realTime\":null,\"routeId\":null,\"routeShortName\":null,\"routeLongName\":null,\"routeType\":null,\"tripBlockId\":null,\"tripId\":null,\"rentedVehicle\":false}]}],\"error\":null}]"

src/test/resources/snapshots/org/opentripplanner/middleware/ConnectedDataPlatform/ConnectedDataPlatformTest/canStreamTheCorrectNumberOfEntitiesDaily-0.json

@@ -0,0 +1 @@
+"[{\"id\":\"bdbac007-43d3-4295-ad97-65e5fad23898\",\"lastUpdated\":1726870100737,\"dateCreated\":1726729200000,\"userId\":\"e5ba1174-7905-4f17-b6ce-d0b77b6b9668\",\"batchId\":\"11111111\",\"fromPlace\":\"Airport, College Park, GA, USA :: 33.64070037704429,-84.44622866991179\",\"toPlace\":\"177 Gibson Street SE, Atlanta, GA, USA :: 33.748893261983575,-84.35611735540574\",\"otp2QueryParams\":{\"arriveBy\":false,\"date\":\"2021-09-22\",\"fromPlace\":\"Airport, College Park, GA, USA :: 33.64070037704429,-84.44622866991179\",\"modes\":[{\"mode\":\"WALK\",\"qualifier\":null},{\"mode\":\"BUS\",\"qualifier\":null},{\"mode\":\"RAIL\",\"qualifier\":null}],\"numItineraries\":0,\"time\":\"15:54\",\"toPlace\":\"177 Gibson Street SE, Atlanta, GA, USA :: 33.748893261983575,-84.35611735540574\",\"walkSpeed\":1.34,\"wheelchair\":false}},{\"id\":\"2930c014-536d-4bb4-a747-25bfa929a781\",\"lastUpdated\":1726870100716,\"dateCreated\":1726729200000,\"userId\":\"e5ba1174-7905-4f17-b6ce-d0b77b6b9668\",\"batchId\":\"99999999\",\"fromPlace\":\"Airport, College Park, GA, USA :: 33.64070037704429,-84.44622866991179\",\"toPlace\":\"177 Gibson Street SE, Atlanta, GA, USA :: 33.748893261983575,-84.35611735540574\",\"otp2QueryParams\":{\"arriveBy\":false,\"date\":\"2021-09-22\",\"fromPlace\":\"Airport, College Park, GA, USA :: 33.64070037704429,-84.44622866991179\",\"modes\":[{\"mode\":\"WALK\",\"qualifier\":null},{\"mode\":\"BUS\",\"qualifier\":null},{\"mode\":\"RAIL\",\"qualifier\":null}],\"numItineraries\":0,\"time\":\"15:54\",\"toPlace\":\"177 Gibson Street SE, Atlanta, GA, USA :: 33.748893261983575,-84.35611735540574\",\"walkSpeed\":1.34,\"wheelchair\":false}}]"