From 3ba001c956695e7845314153a825c082ec543205 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Apr 2020 15:21:25 -0400 Subject: [PATCH 1/4] fix(FeedVersion, gtfsplus.yml): Add call to MTCValidator, change gtfsplus.yml to match MTC guideline --- .../datatools/manager/models/FeedVersion.java | 17 ++++++----------- src/main/resources/gtfs/gtfsplus.yml | 4 ++-- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index de7a1ea86..8e56200b7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -10,6 +10,7 @@ import com.conveyal.gtfs.GTFS; import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.loader.FeedLoadResult; +import com.conveyal.gtfs.validator.MTCValidator; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -226,7 +227,7 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { File gtfsFile; // STEP 1. LOAD GTFS feed into relational database try { - status.update(false,"Unpacking feed...", 15.0); + status.update("Unpacking feed...", 15.0); // Get SQL schema namespace for the feed version. This is needed for reconnecting with feeds // in the database. gtfsFile = retrieveGtfsFile(); @@ -241,9 +242,7 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { this.namespace = feedLoadResult.uniqueIdentifier; LOG.info("Loaded GTFS into SQL {}", feedLoadResult.uniqueIdentifier); } catch (Exception e) { - String errorString = String.format("Error loading GTFS feed for version: %s", this.id); - LOG.warn(errorString, e); - status.update(true, errorString, 0); + status.fail(String.format("Error loading GTFS feed for version: %s", this.id), e); // FIXME: Delete local copy of feed version after failed load? return; } @@ -251,9 +250,7 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { // FIXME: is this the right approach? // if load was unsuccessful, update status and return if(this.feedLoadResult == null) { - String errorString = String.format("Could not load GTFS for FeedVersion %s", id); - LOG.error(errorString); - status.update(true, errorString, 0); + status.fail(String.format("Could not load GTFS for FeedVersion %s", id)); // FIXME: Delete local copy of feed version after failed load? return; } @@ -313,11 +310,9 @@ public void validate(MonitorableJob.Status status) { // run validation on feed version // FIXME: pass status to validate? Or somehow listen to events? status.update("Validating feed...", 33); - validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE); + validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, MTCValidator::new); } catch (Exception e) { - String message = String.format("Unable to validate feed %s", this.id); - LOG.error(message, e); - status.update(true, message, 100, true); + status.fail(String.format("Unable to validate feed %s", this.id), e); // FIXME create validation result with new constructor? validationResult = new ValidationResult(); validationResult.fatalException = "failure!"; diff --git a/src/main/resources/gtfs/gtfsplus.yml b/src/main/resources/gtfs/gtfsplus.yml index 7d085c534..fdb28afb6 100644 --- a/src/main/resources/gtfs/gtfsplus.yml +++ b/src/main/resources/gtfs/gtfsplus.yml @@ -310,7 +310,7 @@ - name: price required: true inputType: TEXT - maxLength: 8 + maxLength: 6 # MTC: (5,2) number format from 0.00 to 999.99. columnWidth: 3 - id: calendar_attributes @@ -338,5 +338,5 @@ - name: zone_name required: true inputType: TEXT - maxLength: 35 + maxLength: 250 helpContent: Public name of the fare zone, as it should appear on 511.org such as EastBay, WestBay, etc From bb2a6d2236150227cf0c3a9286c286c5c20ffa3d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Apr 2020 16:47:51 -0400 Subject: [PATCH 2/4] style(gtfsplus.yml): Order file top-level entries alphabetically. --- src/main/resources/gtfs/gtfsplus.yml | 358 +++++++++++++-------------- 1 file changed, 179 insertions(+), 179 deletions(-) diff --git a/src/main/resources/gtfs/gtfsplus.yml b/src/main/resources/gtfs/gtfsplus.yml index fdb28afb6..55e462fc1 100644 --- a/src/main/resources/gtfs/gtfsplus.yml +++ b/src/main/resources/gtfs/gtfsplus.yml @@ -1,4 +1,128 @@ --- +- id: calendar_attributes + name: calendar_attributes.txt + helpContent: This file contains calendar attributes. This file is needed because standard GTFS does not include these fields. + fields: + - name: service_id + required: true + inputType: GTFS_SERVICE + helpContent: From GTFS calendar.txt file + - name: service_description + required: true + inputType: TEXT + maxLength: 250 + helpContent: Description of the service, as it should appear on 511.org such as Weekdays, Sunday/Holiday + +- id: directions + name: directions.txt + helpContent:

Users of version 1.7 or later must use this file and ignore realtime_directions.txt file.

In the standard GTFS, trip direction can be specified as ‘0’ or ‘1’ only. It does not allow for indication of specific directions such as North, South, etc. This file captures the direction text for each of the directions in a route.

Route direction information can be provided through either this directions.txt OR realtime_directions.txt file, but not both. Users of GTFS+ version 1.7 and above are required to provide directions.txt file and ignore realtime_directions.txt file. Users of version 1.6 or earlier may continue to use realtime_directions.txt, unless they voluntarily choose to upgrade to directions.txt. Only one of directions.txt or realtime_directions.txt should be included. Also, tripheadsign field in GTFS trips.txt file should be populated in order to distinguish patterns within a direction and to provide destination information to passengers.

When directions.txt file is used, optional direction_id field in GTFS trips.txt must be filled in.

+ fields: + - name: route_id + required: true + inputType: GTFS_ROUTE + columnWidth: 6 + helpContent: From GTFS routes.txt file. + - name: direction_id + required: true + inputType: DROPDOWN + options: + - value: '0' + - value: '1' + columnWidth: 3 + helpContent: Binary direction_id from GTFS trips.txt file. Each (route_id, direction_id) pair can only appear once in directions.txt. + - name: direction + required: true + inputType: DROPDOWN + options: + - value: North + - value: South + - value: East + - value: West + - value: Northeast + - value: Northwest + - value: Southeast + - value: Southwest + - value: Clockwise + - value: Counterclockwise + - value: Inbound + - value: Outbound + - value: Loop + - value: A Loop + - value: B Loop + columnWidth: 3 + helpContent: Corresponding direction name. + +- id: fare_rider_categories + name: fare_rider_categories.txt + helpContent: This file specifies attributes for the fares for rider categories. GTFS file fare_attributes.txt contains the fares for the Regular rider category. Fares for other rider categories such as Child, Senior, etc will be provided in this plus file fare_rider_categories.txt. The combination of fare_id and rider_category_id should be unique in this file. This file is needed because standard GTFS does not include these fields. + fields: + - name: fare_id + required: true + inputType: GTFS_FARE + columnWidth: 6 + helpContent: From GTFS fare_attributes.txt file + - name: rider_category_id + required: true + inputType: DROPDOWN + options: + - value: '2' + text: Senior + - value: '3' + text: Child + - value: '4' + text: Student + - value: '5' + text: Youth + - value: '6' + text: Disabled + - value: '7' + text: Promotional category + - value: '11' + text: Military + - value: '15' + text: Custom (15) + - value: '16' + text: Custom (16) + - value: '17' + text: Custom (17) + - value: '18' + text: Custom (18) + - value: '19' + text: Custom (19) + - value: '20' + text: Custom (20) + - value: '21' + text: Custom (21) + - value: '22' + text: Custom (22) + - value: '23' + text: Custom (23) + - value: '24' + text: Custom (24) + - value: '25' + text: Custom (25) + columnWidth: 3 + helpContent: From Plus file rider_categories.txt + - name: price + required: true + inputType: TEXT + maxLength: 6 # MTC: (5,2) number format from 0.00 to 999.99. + columnWidth: 3 + +- id: farezone_attributes + name: farezone_attributes.txt + helpContent: This file contains fare zone attributes. This file is needed because standard GTFS does not include these fields. + fields: + - name: zone_id + required: true + inputType: GTFS_ZONE + helpContent: From GTFS stops.txt file + - name: zone_name + required: true + inputType: TEXT + maxLength: 250 + helpContent: Public name of the fare zone, as it should appear on 511.org such as EastBay, WestBay, etc + - id: realtime_routes name: realtime_routes.txt helpContent: This file contains additional real-time information for a route. @@ -49,45 +173,6 @@ columnWidth: 2 helpContent: Corresponding stop_code used in real-time feed for MTC. -- id: directions - name: directions.txt - helpContent:

Users of version 1.7 or later must use this file and ignore realtime_directions.txt file.

In the standard GTFS, trip direction can be specified as ‘0’ or ‘1’ only. It does not allow for indication of specific directions such as North, South, etc. This file captures the direction text for each of the directions in a route.

Route direction information can be provided through either this directions.txt OR realtime_directions.txt file, but not both. Users of GTFS+ version 1.7 and above are required to provide directions.txt file and ignore realtime_directions.txt file. Users of version 1.6 or earlier may continue to use realtime_directions.txt, unless they voluntarily choose to upgrade to directions.txt. Only one of directions.txt or realtime_directions.txt should be included. Also, tripheadsign field in GTFS trips.txt file should be populated in order to distinguish patterns within a direction and to provide destination information to passengers.

When directions.txt file is used, optional direction_id field in GTFS trips.txt must be filled in.

- fields: - - name: route_id - required: true - inputType: GTFS_ROUTE - columnWidth: 6 - helpContent: From GTFS routes.txt file. - - name: direction_id - required: true - inputType: DROPDOWN - options: - - value: '0' - - value: '1' - columnWidth: 3 - helpContent: Binary direction_id from GTFS trips.txt file. Each (route_id, direction_id) pair can only appear once in directions.txt. - - name: direction - required: true - inputType: DROPDOWN - options: - - value: North - - value: South - - value: East - - value: West - - value: Northeast - - value: Northwest - - value: Southeast - - value: Southwest - - value: Clockwise - - value: Counterclockwise - - value: Inbound - - value: Outbound - - value: Loop - - value: A Loop - - value: B Loop - columnWidth: 3 - helpContent: Corresponding direction name. - - id: realtime_trips name: realtime_trips.txt helpContent: This file contains real-time identifiers for trips. This file can be omitted if the GTFS trips.txt file contains the same Trip IDs included in real-time feed for MTC within the trip_id field. @@ -104,6 +189,61 @@ columnWidth: 6 helpContent: Corresponding trip_id provided in real-time feed for MTC. +- id: rider_categories + name: rider_categories.txt + helpContent: This file lists the rider categories for fares other than the Regular category. This file is needed because standard GTFS does not include these fields. + fields: + - name: rider_category_id + required: true + inputType: DROPDOWN + options: + - value: '1' + text: Adult + - value: '2' + text: Senior + - value: '3' + text: Child + - value: '4' + text: Student + - value: '5' + text: Youth + - value: '6' + text: Disabled + - value: '7' + text: Promotional category + - value: '11' + text: Military + - value: '15' + text: Custom (15) + - value: '16' + text: Custom (16) + - value: '17' + text: Custom (17) + - value: '18' + text: Custom (18) + - value: '19' + text: Custom (19) + - value: '20' + text: Custom (20) + - value: '21' + text: Custom (21) + - value: '22' + text: Custom (22) + - value: '23' + text: Custom (23) + - value: '24' + text: Custom (24) + - value: '25' + text: Custom (25) + columnWidth: 6 + helpContent: Unique rider category ID (agency can assign categories that do not fall under standard categories) + - name: rider_category_description + required: true + inputType: TEXT + maxLength: 256 + columnWidth: 6 + helpContent: Rider category as it should appear on 511.org, such as Child (ages 5-11), Seniors (Ages 62 & Up) + - id: stop_attributes name: stop_attributes.txt helpContent: This file contains additional attributes for a stop. This file is needed because standard GTFS does not include these fields. @@ -200,143 +340,3 @@ inputType: GTFS_STOP columnWidth: 6 helpContent: From GTFS stop_times.txt file (Include Time point stops only) - -- id: rider_categories - name: rider_categories.txt - helpContent: This file lists the rider categories for fares other than the Regular category. This file is needed because standard GTFS does not include these fields. - fields: - - name: rider_category_id - required: true - inputType: DROPDOWN - options: - - value: '1' - text: Adult - - value: '2' - text: Senior - - value: '3' - text: Child - - value: '4' - text: Student - - value: '5' - text: Youth - - value: '6' - text: Disabled - - value: '7' - text: Promotional category - - value: '11' - text: Military - - value: '15' - text: Custom (15) - - value: '16' - text: Custom (16) - - value: '17' - text: Custom (17) - - value: '18' - text: Custom (18) - - value: '19' - text: Custom (19) - - value: '20' - text: Custom (20) - - value: '21' - text: Custom (21) - - value: '22' - text: Custom (22) - - value: '23' - text: Custom (23) - - value: '24' - text: Custom (24) - - value: '25' - text: Custom (25) - columnWidth: 6 - helpContent: Unique rider category ID (agency can assign categories that do not fall under standard categories) - - name: rider_category_description - required: true - inputType: TEXT - maxLength: 256 - columnWidth: 6 - helpContent: Rider category as it should appear on 511.org, such as Child (ages 5-11), Seniors (Ages 62 & Up) - -- id: fare_rider_categories - name: fare_rider_categories.txt - helpContent: This file specifies attributes for the fares for rider categories. GTFS file fare_attributes.txt contains the fares for the Regular rider category. Fares for other rider categories such as Child, Senior, etc will be provided in this plus file fare_rider_categories.txt. The combination of fare_id and rider_category_id should be unique in this file. This file is needed because standard GTFS does not include these fields. - fields: - - name: fare_id - required: true - inputType: GTFS_FARE - columnWidth: 6 - helpContent: From GTFS fare_attributes.txt file - - name: rider_category_id - required: true - inputType: DROPDOWN - options: - - value: '2' - text: Senior - - value: '3' - text: Child - - value: '4' - text: Student - - value: '5' - text: Youth - - value: '6' - text: Disabled - - value: '7' - text: Promotional category - - value: '11' - text: Military - - value: '15' - text: Custom (15) - - value: '16' - text: Custom (16) - - value: '17' - text: Custom (17) - - value: '18' - text: Custom (18) - - value: '19' - text: Custom (19) - - value: '20' - text: Custom (20) - - value: '21' - text: Custom (21) - - value: '22' - text: Custom (22) - - value: '23' - text: Custom (23) - - value: '24' - text: Custom (24) - - value: '25' - text: Custom (25) - columnWidth: 3 - helpContent: From Plus file rider_categories.txt - - name: price - required: true - inputType: TEXT - maxLength: 6 # MTC: (5,2) number format from 0.00 to 999.99. - columnWidth: 3 - -- id: calendar_attributes - name: calendar_attributes.txt - helpContent: This file contains calendar attributes. This file is needed because standard GTFS does not include these fields. - fields: - - name: service_id - required: true - inputType: GTFS_SERVICE - helpContent: From GTFS calendar.txt file - - name: service_description - required: true - inputType: TEXT - maxLength: 250 - helpContent: Description of the service, as it should appear on 511.org such as Weekdays, Sunday/Holiday - -- id: farezone_attributes - name: farezone_attributes.txt - helpContent: This file contains fare zone attributes. This file is needed because standard GTFS does not include these fields. - fields: - - name: zone_id - required: true - inputType: GTFS_ZONE - helpContent: From GTFS stops.txt file - - name: zone_name - required: true - inputType: TEXT - maxLength: 250 - helpContent: Public name of the fare zone, as it should appear on 511.org such as EastBay, WestBay, etc From 09923172915d772f1dad7b6e2d8892fdda98a5c9 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 14 Apr 2020 11:22:54 -0400 Subject: [PATCH 3/4] fix(FeedVersion): Run MTCValidator only if 'mtc' extension is enabled. --- .../datatools/manager/models/FeedVersion.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 8e56200b7..1da24005d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.UUID; +import static com.conveyal.datatools.manager.DataManager.isExtensionEnabled; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -307,10 +308,16 @@ public void validate(MonitorableJob.Status status) { // VALIDATE GTFS feed try { LOG.info("Beginning validation..."); - // run validation on feed version // FIXME: pass status to validate? Or somehow listen to events? status.update("Validating feed...", 33); - validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, MTCValidator::new); + + // Validate the feed version. + // Certain extensions, if enabled, have extra validators + if (isExtensionEnabled("mtc")) { + validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, MTCValidator::new); + } else { + validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE); + } } catch (Exception e) { status.fail(String.format("Unable to validate feed %s", this.id), e); // FIXME create validation result with new constructor? From 0ae68ed650c4805d8db3a8205276980d0118103d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 14 Apr 2020 11:35:58 -0400 Subject: [PATCH 4/4] build(maven): Update gtfs-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 49aade49a..9ebe2660d 100644 --- a/pom.xml +++ b/pom.xml @@ -251,7 +251,7 @@ com.conveyal gtfs-lib - 5.1.0 + 6.0.0