Merge pull request #629 from ibi-group/feature/DT-526-skip-non-revenue-if-fails

Skip non revenue transformation if it fails

Author: binh-dam-ibigroup

src/main/java/com/conveyal/datatools/manager/models/transform/RemoveNonRevenueTripsTransformation.java

@@ -36,6 +36,9 @@
  * This transformation removes non revenue trips from a feed. Establishing trips where _all_ related stop times have
  * "no pickup available" or "no continuous stopping pickup". Then remove these stop times and related trips from the
  * feed. "Non revenue" stops are kept because the transit ops systems use these in the published feed for all trips.
+ *
+ * If the transformation fails for any reason e.g. because of missing fields, the transformation is skipped instead of
+ * being failed. This allows subsequent transformations to take place and for the import to still succeed.
  */
 public class RemoveNonRevenueTripsTransformation extends ZipTransformation {
     private static final Logger LOG = LoggerFactory.getLogger(RemoveNonRevenueTripsTransformation.class);
@@ -60,8 +63,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st
             Field[] fieldsFoundInStopTimes = gtfsTable.getFieldsFromFieldHeaders(headersForStopTime, null);
             Map<String, Integer> fieldIndexes = getFieldIndexes(fieldsFoundInStopTimes);
             if (!hasRequiredFields(fieldIndexes)) {
-                status.error = true;
-                status.fail("Unable to remove non revenue trips because the stop times file does not contain all required fields.");
+                LOG.warn("Unable to remove non revenue trips because the stop times file does not contain all required fields.");
                 return;
             }
 
@@ -76,8 +78,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st
             Field[] fieldsFoundInStopTrips = gtfsTable.getFieldsFromFieldHeaders(headersForTrips, null);
             int tripIdFieldIndex = getFieldIndex(fieldsFoundInStopTrips, TRIP_ID_FIELD_NAME);
             if (tripIdFieldIndex == -1) {
-                status.error = true;
-                status.fail("Unable to remove non revenue trips because the trips file does not contain the required trip id field.");
+                LOG.warn("Unable to remove non revenue trips because the trips file does not contain the required trip id field.");
                 return;
             }
             RevenueData revenueTrips = getAllRevenueTrips(csvReaderForTrips, revenueStopTimes.getStopTimeRows().keySet(), tripIdFieldIndex);
@@ -101,7 +102,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st
                 revenueTrips.getDeletedRowCount()
             );
         } catch (Exception e) {
-            status.fail("Unknown error encountered while attempting to remove non revenue trip from zip file.", e);
+            LOG.error("Unknown error encountered while attempting to remove non revenue trip from zip file.", e);
         }
     }
 

src/test/java/com/conveyal/datatools/manager/models/transform/RemoveNonRevenueTripsTransformationTest.java

@@ -158,6 +158,17 @@ void testDirectNonRevenueTripRemovalTransformation() throws Exception {
         assertFalse(hasNonRevenueTrips(zipTarget.gtfsFile.getAbsolutePath()));
     }
 
+    @Test
+    void testDirectNonRevenueTripRemovalTransformationFail() throws Exception {
+        // Feed is missing required pickup and drop off fields. This will stop the transformation, but should not flag
+        // it as failed. Subsequent transformations and import can then still take place.
+        File zip = zipFolderFiles("non-revenue-trips-fail");
+        FeedTransformZipTarget zipTarget = new FeedTransformZipTarget(zip);
+        MonitorableJob.Status status = new MonitorableJob.Status();
+        trans.transform(zipTarget, status);
+        assertFalse(status.error);
+    }
+
     private void hadExpectTransformResults(List<TableTransformResult> tableTransformResults) {
         assertEquals(2, tableTransformResults.size());
         assertEquals(5, tableTransformResults.get(0).deletedCount);

src/test/resources/com/conveyal/datatools/gtfs/non-revenue-trips-fail/stop_times.txt

@@ -0,0 +1,10 @@
+trip_id,arrival_time,departure_time,stop_id,continuous_pickup,continuous_drop_off
+non-revenue-trip,04:58:00,04:58:00,CONC,1,1
+non-revenue-trip,05:03:00,05:03:00,PHIL,1,1
+non-revenue-trip,05:06:00,05:06:00,WCRK,1,1
+non-revenue-trip,05:11:00,05:11:00,LAFY,1,1
+non-revenue-trip,05:12:00,05:13:00,TGLD,1,1
+revenue-trip,04:58:00,04:58:00,CONC,0,0
+revenue-trip,05:03:00,05:03:00,PHIL,1,1
+revenue-trip,05:06:00,05:06:00,WCRK,1,1
+revenue-trip,05:11:00,05:11:00,LAFY,0,0