Merge pull request #141 from conveyal/some-stuff

Export improvements, proper schedule_exception handling and more tests

Author: Landon Reed

.travis.yml

@@ -4,6 +4,9 @@ language: java
 jdk:
   - oraclejdk8
 
+addons:
+  postgresql: "9.6"
+
 # Replace Travis's default Maven installation step with a no-op.
 # This avoids redundantly pre-running 'mvn install -DskipTests' every time.
 install: true

README.md

@@ -16,7 +16,7 @@ A gtfs-lib GTFSFeed object should faithfully represent the contents of a single
 
 ## Usage
 
-gtfs-lib can be used as a Java library or run via the command line.
+gtfs-lib can be used as a Java library or run via the command line.  If using this library with PostgreSQL for persistence, you must use at least version 9.6 of PostgreSQL.
 
 ### Library (maven)
 

src/main/java/com/conveyal/gtfs/loader/BatchTracker.java

@@ -0,0 +1,53 @@
+package com.conveyal.gtfs.loader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Avoid Java's "effectively final" nonsense when using prepared statements in foreach loops.
+ * Automatically push execute batches of prepared statements before the batch gets too big.
+ * TODO there's probably something like this in an Apache Commons util library
+ */
+public class BatchTracker {
+    private static final Logger LOG = LoggerFactory.getLogger(BatchTracker.class);
+
+    private final String recordType;
+    private PreparedStatement preparedStatement;
+    private int currentBatchSize = 0;
+    private int totalRecordsProcessed = 0;
+
+    public BatchTracker(String recordType, PreparedStatement preparedStatement) {
+        this.preparedStatement = preparedStatement;
+        this.recordType = recordType;
+    }
+
+    public void addBatch() throws SQLException {
+        preparedStatement.addBatch();
+        currentBatchSize += 1;
+        if (currentBatchSize > JdbcGtfsLoader.INSERT_BATCH_SIZE) {
+            preparedStatement.executeBatch();
+            totalRecordsProcessed += currentBatchSize;
+            currentBatchSize = 0;
+        }
+    }
+
+    public void executeRemaining() throws SQLException {
+        if (currentBatchSize > 0) {
+            totalRecordsProcessed += currentBatchSize;
+            preparedStatement.executeBatch();
+            currentBatchSize = 0;
+        }
+        // Avoid reuse, signal that this was cleanly closed.
+        preparedStatement = null;
+        LOG.info(String.format("Inserted %d %s records", totalRecordsProcessed, recordType));
+    }
+
+    public void finalize () {
+        if (preparedStatement != null || currentBatchSize > 0) {
+            throw new RuntimeException("BUG: It looks like someone did not call executeRemaining on a BatchTracker.");
+        }
+    }
+}

src/main/java/com/conveyal/gtfs/loader/DoubleField.java

@@ -1,6 +1,5 @@
 package com.conveyal.gtfs.loader;
 
-import com.conveyal.gtfs.error.NewGTFSErrorType;
 import com.conveyal.gtfs.storage.StorageException;
 
 import java.sql.JDBCType;
@@ -17,13 +16,17 @@
 public class DoubleField extends Field {
 
     private double minValue;
-
     private double maxValue;
+    // This field dictates how many decimal places this field should be rounded to when exporting to a GTFS.
+    // The place where the rounding happens during exports is in Table.commaSeparatedNames.
+    // A value less than 0 indicates that no rounding should happen.
+    private int outputPrecision;
 
-    public DoubleField (String name, Requirement requirement, double minValue, double maxValue) {
+    public DoubleField (String name, Requirement requirement, double minValue, double maxValue, int outputPrecision) {
         super(name, requirement);
         this.minValue = minValue;
         this.maxValue = maxValue;
+        this.outputPrecision = outputPrecision;
     }
 
     private double validate(String string) {
@@ -63,4 +66,20 @@ public String getSqlTypeName () {
         return "double precision";
     }
 
+    /**
+     * When outputting to csv, round fields that have been created with an outputPrecision > -1 to avoid excessive
+     * precision.
+     */
+    @Override
+    public String getColumnExpression(String prefix, boolean csvOutput) {
+        String columnName = super.getColumnExpression(prefix, csvOutput);
+        if (!csvOutput || this.outputPrecision < 0) return columnName;
+        return String.format(
+            "round(%s::DECIMAL, %d) as %s",
+            columnName,
+            this.outputPrecision,
+            name
+        );
+    }
+
 }

src/main/java/com/conveyal/gtfs/loader/Field.java

@@ -140,4 +140,13 @@ public Field permitEmptyValue () {
     public boolean isEmptyValuePermitted() {
         return this.emptyValuePermitted;
     }
+
+    /**
+     * Get the expression used to select this column from the database based on the prefix.  The csvOutput parameter is
+     * needed in overriden method implementations that have special ways of outputting certain fields.  The prefix
+     * parameter is assumed to be either null or a string in the format: `schema.`
+     */
+    public String getColumnExpression(String prefix, boolean csvOutput) {
+        return prefix != null ? String.format("%s%s", prefix, name) : name;
+    }
 }