Merge pull request #57 from gtracy/gtfs-refresh

Implemented a getstops endpoint in the API
gtracy · May 13, 2023 · 00dab3f · 00dab3f
2 parents d642855 + 62e8e9f
commit 00dab3f
Show file tree

Hide file tree

Showing 6 changed files with 121 additions and 41 deletions.
diff --git a/api/stops.js b/api/stops.js
@@ -1,30 +1,36 @@
 'use strict';
 
 const moment = require('moment-timezone');
+const fs = require('fs');
 const devkey = require('./devkey');
 const utils = require('./utils');
-const Stop = require('../lib/stops');
+const gtfs_stop = require('../lib/stops');
 
 const config = require('../config');
 const logger = require('pino')(config.getLogConfig());
 
 
+
 module.exports = async function(app) {
 
     // {
     //     "status" : "0",
     //     "timestamp" : "12:38pm",
     //     "stopID" : "1391",
+    //     "stop_code" : "1391",
     //     "intersection" : "Atwood & Ohio",
     //     "latitude" : "43.0937715",
     //     "longitude" : "-89.3467281",
     // }
-    app.get('/v1/getstoplocation', utils.validateRequest, utils.afterHours, devkey.validateDevKey, async (req,res) => {
+    app.get('/v1/getstoplocation', utils.validateRequest, devkey.validateDevKey, async (req,res) => {
         var json_result = {};
-        var gtfs_stop = new Stop();
+
+        function removeLeadingZeros(str) {
+            return str.replace(/^0+/, '');
+        }          
 
         // snag the API query details
-        const stop_id = req.query.stopID;
+        const stop_id = removeLeadingZeros(req.query.stopID);
 
         json_result.status = "0";
         json_result.timestamp = moment().tz("America/Chicago").format("h:mmA");
@@ -34,6 +40,7 @@ module.exports = async function(app) {
         const stop_data = await gtfs_stop.fetchById(stop_id);
         if( stop_data ) {
             // inspect results and build the payload
+            json_result.stop_code = stop_data.stop_code;
             json_result.intersection = stop_data.stop_name;
             json_result.latitude = parseFloat(stop_data.stop_lat);
             json_result.longitude = parseFloat(stop_data.stop_lon);
@@ -45,19 +52,28 @@ module.exports = async function(app) {
         res.json(json_result);
     });
 
-    app.get('/v1/getstops',  (req,res) => {
-        res.json({
-            status : 0,
-            description : "this endpoint has been deprecated"
-        });
+    app.get('/v1/getstops', devkey.validateDevKey, async (req,res) => {
+        var json_result = {};
+
+        json_result.status = "0";
+        json_result.timestamp = moment().tz("America/Chicago").format("h:mmA");
+
+        // the stop data is static so we grab from a json file
+        // to simplify the implementation
+        const data = fs.readFileSync('./lib/stops.json', 'utf8');
+        const stop_json = JSON.parse(data);
+        json_result.stops = stop_json;
+
+        logger.debug(json_result,'/v1/getstops ');
+        res.json(json_result);
     });
 
+
     app.get('/v1/getnearbystops',  (req,res) => {
         res.json({
             status : 0,
             description : "this endpoint has been deprecated"
         });
     });
 
-
 }
diff --git a/lib/stops.js b/lib/stops.js
@@ -3,13 +3,15 @@
 const _ = require('underscore');
 const config = require('../config');
 const logger = require('pino')(config.getLogConfig());
+const fs = require("fs");
+const csvToJson = require('convert-csv-to-json');
 
 let AWS = require('aws-sdk');
 
 //
 // overloaded object that encapsulates GTFS Stop details
 //
-function Stop() {
+function _Stops() {
     const TABLE_NAME = 'Stops_gtfs';
 
     // Create an Amazon DynamoDB service client object.
@@ -20,8 +22,8 @@ function Stop() {
         return TABLE_NAME;
     }
 
-    // fetch Trip details by tripId
-    //    trip_ids is an array of GTFS trip id strings
+    // fetch Stop details by stopID
+    //    object representing a stop's details
     //
     this.fetchById = async (stop_id) => {
         let descriptions = [];
@@ -41,7 +43,8 @@ function Stop() {
 
                 if( aws_result.Item) {
                     return {
-                        stop_id : aws_result.Item.stop_id.S,
+                        stop_id : aws_result.Item.stop_code.S,
+                        stop_code : aws_result.Item.stop_code.S,
                         stop_name : aws_result.Item.stop_name.S,
                         stop_lat : aws_result.Item.stop_lat.S,
                         stop_lon : aws_result.Item.stop_lon.S
@@ -59,8 +62,67 @@ function Stop() {
         }
     }
 
+    function getIntersection(metro_intersection) {
+        let index = metro_intersection.indexOf('(');
+        if (index !== -1) {
+            return metro_intersection.slice(0, index);
+        } else {
+            return metro_intersection;
+        }
+    }
+
+    function getDirection(metro_intersection) {
+        const regex = /\(([^)]+)\)/;
+        const match = regex.exec(metro_intersection);
+        if (match) {
+            switch( match[1] ) {
+                case 'EB':
+                    return "Eastbound";
+                    break;
+                case 'WB':
+                    return "Westbound";
+                    break;
+                case 'NB':
+                    return "Northbound";
+                    break;
+                case 'SB':
+                    return "Southbound";
+                    break;
+                default:
+                    return match[1];
+            }
+        }
+        return "";      
+    }
+
+    // utility function that transforms and shrink a GTFS Stops 
+    // file into a static JSON object
+    this.transformGTFSFileToJSON = (gtfs_input_file,json_output_file) => {
+
+        let json = csvToJson.fieldDelimiter(',').getJsonFromCsv(gtfs_input_file);
+        console.log('total stops : '+json.length);
+        console.log('sample stop transformed : ');
+        console.log(json[0]);
+        let json_small = _.map(json, (obj) => {
+            return ({
+                'stopID' : obj.stop_id,
+                'stop_code' : obj.stop_code,
+                'intersection' : getIntersection(obj.stop_name).trim(),
+                'lat' : obj.stop_lat,
+                'lon' : obj.stop_lon,
+                'direction' : getDirection(obj.stop_name),
+            });
+        });
+        console.dir(json_small[0]);
+        fs.writeFile(json_output_file, JSON.stringify(json_small), (error) => {
+            if (error) throw error;
+        });
+    }
+
+
 }
-module.exports = Stop;
+let stops = new _Stops();
+module.exports = stops;
 
 
 
diff --git a/lib/stops.json b/lib/stops.json
diff --git a/public/locations.html b/public/locations.html
@@ -195,7 +195,7 @@ <h1><a name="getstops">getstops</a></h1>
  <p class="deprecated">this enpoint is deprecated</p>
  <div id="spec">
  <h3>Overview</h3>
- The getstops method will return the geo locations for all stops of a specified route.
+ The getstops method will return the labels and geo locations for all stops in the system.
  <h3>URL</h3>
  https://api.smsmybus.com/v1/getstops
  <h3>Fields</h3>
@@ -210,45 +210,34 @@ <h3>Fields</h3>
   <td class="center">String (required)</td>
   <td>API access key</td>
  </tr>
- <tr class="striped">
-  <td class="center">routeID</td>
-  <td class="center">String</td>
-  <td>A two-digit value describing a valid route in the Metro system.</td>
- </tr>
- <tr>
-  <td class="center">destination</td>
-  <td class="center">String</td>
-  <td>A value describing a valid route endpoint in the Metro system. When a destination is included in the query, the results will be limited to the stops that are headed in that direction.</td>
- </tr>
  </table>
  <h3>Examples</h3>
  Retrieve all stops for route 3:<p> </p>
- <span style="font-weight:bold;margin-left:24px;">https://api.smsmybus.com/v1/getstops?key=xxx&routeID=03</span><p> </p>
- Retrieve all stops for route 3 headed toward ETP:<p> </p>
- <span style="font-weight:bold;margin-left:24px;">https://api.smsmybus.com/v1/getstops?key=xxx&routeID=03&destination=ETP</span><p> </p>
+ <span style="font-weight:bold;margin-left:24px;">https://api.smsmybus.com/v1/getstops?key=xxx</span><p> </p>
  <h3>Response</h3>
  The response length is dependent on the particular request parameters. If the status field is 0, the response
- will be grouped by route, listing the details of each stop on that route.
+ will list the details of all stops in the system.
  <p> </p>
  <p>No assumptions should be made about the order of the stops in the response.</p>
 <code class="prettyprint lang-javascript">
 {
     "status" : "0",
     "timestamp" : "12:38pm",
-    "routeID" : "03",
     "stops" : [
         {
           "stopID" : "1391",
+          "stop_code" : "1391",
           "intersection" : "Atwood & Ohio",
           "latitude" : "43.0937715",
           "longitude" : "-89.3467281",
           "destination" : "ETP",
         },
         {
-          "stopID" : "1961",
-          "intersection" : "Atwood & Rusk",
-          "latitude" : "43.0924936",
-          "longitude" : "-89.3528187",
+          "stopID" : "43",
+          "stop_code" : "0043",
+          "intersection" : "W Dayton & N Lake",
+          "latitude":"43.070811",
+          "longitude":"-89.397884"
           "destination" : "ETP",
         },
         ...

diff --git a/utils/gtfs.js b/utils/gtfs.js
@@ -100,9 +100,16 @@ module.exports.gtfsToDynamo = async (AWS,gtfs_file,dynamo_table,table_params) =>
             [dynamo_table]: putReqs
             }
         }
-
-        await dynamoClient.batchWrite(req).promise();
-        console.log('  batch of ' + batch.length + ' written to dynamo');
+
+        try {
+            await dynamoClient.batchWrite(req).promise();
+            console.log('  batch of ' + batch.length + ' written to dynamo');
+        } catch(error) {
+            console.log('Dynamo write failed - ' + error);
+            console.dir(putReqs[0].PutRequest.Item);
+            console.dir(putReqs[1].PutRequest.Item);
+            throw error;
+        }
     }
 
 }
diff --git a/utils/stop_import.js b/utils/stop_import.js
@@ -11,6 +11,7 @@
 //
 let gtfs = require('./gtfs');
 let config = require('../config');
+const Stops = require('../lib/stops');
 
 let AWS = require('aws-sdk');
 AWS.config.update(config.getAWSConfig())
@@ -35,14 +36,18 @@ console.dir(config.getAWSConfig());
             WriteCapacityUnits: 25
         }
     };
-
-    // import the GTFS file into our new table
+
     try{
         let input_file = "gtfs/stops.txt";
         // test if user is parsing a non-default GTFS file
         if( process.argv[2] ) {
             input_file = process.argv[2];
-        }    
+        }
+        // export stops into a json object
+        let json_output = './lib/stops.json';    
+        await Stops.transformGTFSFileToJSON(input_file,json_output);
+
+        // push stop details into Dynamo
         await gtfs.gtfsToDynamo(AWS,input_file,TABLE_NAME,params);
     } catch(err) {
         console.log('import failed!');