Merge pull request #121 from frodrigo/enableKeepPartialLinestring

Add option enableKeepPartialLinestring to generate anyway partial geometry when missing/invalid nodes are found

doc/detailed-usage.adoc

@@ -1680,6 +1680,12 @@ improvements compared to the query approach. |yes, no |no
 |enableLinestringBuilder |As per the enableBboxBuilder option but for
 the linestring geometry column. |yes, no |no
 
+|enableKeepPartialLinestring |This option affects how linestrings are
+built from option enableLinestringBuilder. When an invalid or a missing
+node location is encountered the linestring is not built by default.
+Enabling this option keeps it as a partial linestring. It will result in
+a different geometry than the original one. |yes, no |no
+
 |nodeLocationStoreType |This option only takes effect if at least one of
 the enableBboxBuilder and enableLinestringBuilder options are enabled.
 Both geometry builder implementations require knowledge of all node
@@ -1891,6 +1897,12 @@ much slower but still faster than relying on the default database
 geometry building implementation, or the "CompactTempFile" option which
 is more efficient for smaller datasets. |"InMemory", "TempFile",
 "CompactTempFile" |"CompactTempFile"
+
+|enableKeepPartialLinestring |This option affects how linestrings are
+built. When an invalid or a missing node location is encountered the
+linestring is not built by default. Enabling this option keeps it as a
+partial linestring. It will result in a different geometry than the
+original one. |yes, no |no
 |=======================================================================
 
 ==== --write-pgsimp-dump (--wsd)
@@ -1923,6 +1935,12 @@ improvements compared to the query approach. |yes, no |no
 |enableLinestringBuilder |As per the enableBboxBuilder option but for
 the linestring geometry column. |yes, no |no
 
+|enableKeepPartialLinestring |This option affects how linestrings are
+built from option enableLinestringBuilder. When an invalid or a missing
+node location is encountered the linestring is not built by default.
+Enabling this option keeps it as a partial linestring. It will result in
+a different geometry than the original one. |yes, no |no
+
 |nodeLocationStoreType |This option only takes effect if at least one of
 the enableBboxBuilder and enableLinestringBuilder options are enabled.
 Both geometry builder implementations require knowledge of all node
@@ -1934,6 +1952,12 @@ you may use the "InMemory" option. Otherwise you must choose between the
 the default database geometry building implementation, or the
 "CompactTempFile" option which is more efficient for smaller datasets.
 |"InMemory", "TempFile", "CompactTempFile" |"CompactTempFile"
+
+|enableKeepPartialLinestring |This option affects how linestrings are
+built. When an invalid or a missing node location is encountered the
+linestring is not built by default. Enabling this option keeps it as a
+partial linestring. It will result in a different geometry than the
+original one. |yes, no |no
 |=======================================================================
 
 ==== --truncate-pgsimp (--ts)

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriter.java

@@ -34,6 +34,7 @@ public class PostgreSqlCopyWriter implements Sink {
 	private NodeLocationStoreType storeType;
 	private boolean populateBbox;
 	private boolean populateLinestring;
+	private boolean enableKeepPartialLinestring;
 	private boolean initialized;
 
 
@@ -46,13 +47,17 @@ public class PostgreSqlCopyWriter implements Sink {
 	 *            Contains preferences configuring database behaviour.
 	 * @param storeType
 	 *            The node location storage type used by the geometry builders.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 */
 	public PostgreSqlCopyWriter(
 			DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences,
-			NodeLocationStoreType storeType) {
+			NodeLocationStoreType storeType, boolean enableKeepPartialLinestring) {
 		this.loginCredentials = loginCredentials;
 		this.preferences = preferences;
 		this.storeType = storeType;
+		this.enableKeepPartialLinestring = enableKeepPartialLinestring;
 
 		copyFileset = new TempCopyFileset();
 	}
@@ -69,7 +74,8 @@ private void initialize() {
 				populateLinestring = capabilityChecker.isWayLinestringSupported();				
 
 				copyFilesetBuilder =
-					new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, storeType);
+					new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, enableKeepPartialLinestring,
+						storeType);
 
 				copyFilesetLoader = new CopyFilesetLoader(loginCredentials, preferences, copyFileset);
 

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriterFactory.java

@@ -18,6 +18,8 @@
 public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory {
 	private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType";
 	private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile";
+	private static final String ARG_ENABLE_KEEP_PARTIAL_LIENSTRING = "enableKeepPartialLinestring";
+	private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING = false;
 
 	/**
 	 * {@inheritDoc}
@@ -27,17 +29,20 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 		DatabaseLoginCredentials loginCredentials;
 		DatabasePreferences preferences;
 		NodeLocationStoreType storeType;
+		boolean enableKeepPartialLinestring;
 
 		// Get the task arguments.
 		loginCredentials = getDatabaseLoginCredentials(taskConfig);
 		preferences = getDatabasePreferences(taskConfig);
 		storeType = Enum.valueOf(
 				NodeLocationStoreType.class,
 				getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE));
+		enableKeepPartialLinestring = getBooleanArgument(taskConfig, ARG_ENABLE_KEEP_PARTIAL_LIENSTRING,
+			DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING);
 
 		return new SinkManager(
 			taskConfig.getId(),
-			new PostgreSqlCopyWriter(loginCredentials, preferences,	storeType),
+			new PostgreSqlCopyWriter(loginCredentials, preferences,	storeType, enableKeepPartialLinestring),
 			taskConfig.getPipeArgs()
 		);
 	}

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriter.java

@@ -37,18 +37,23 @@ public class PostgreSqlDumpWriter implements Sink {
 	 *            processing instead of relying on the database to build them
 	 *            after import. This increases processing but is faster than
 	 *            relying on the database.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 * @param storeType
 	 *            The node location storage type used by the geometry builders.
 	 */
 	public PostgreSqlDumpWriter(
 			File filePrefix, boolean enableBboxBuilder,
-			boolean enableLinestringBuilder, NodeLocationStoreType storeType) {
+			boolean enableLinestringBuilder, boolean enableKeepPartialLinestring,
+			NodeLocationStoreType storeType) {
 		DirectoryCopyFileset copyFileset;
 
 		copyFileset = new DirectoryCopyFileset(filePrefix);
 
 		copyFilesetBuilder =
-			new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder, storeType);
+			new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder,
+				enableKeepPartialLinestring, storeType);
 	}
 
 

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriterFactory.java

@@ -18,10 +18,12 @@
 public class PostgreSqlDumpWriterFactory extends TaskManagerFactory {
 	private static final String ARG_ENABLE_BBOX_BUILDER = "enableBboxBuilder";
 	private static final String ARG_ENABLE_LINESTRING_BUILDER = "enableLinestringBuilder";
+	private static final String ARG_ENABLE_KEEP_PARTIAL_LINESTRING = "enableKeepPartialLinestring";
 	private static final String ARG_FILE_NAME = "directory";
 	private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType";
 	private static final boolean DEFAULT_ENABLE_BBOX_BUILDER = false;
 	private static final boolean DEFAULT_ENABLE_LINESTRING_BUILDER = false;
+	private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING = false;
 	private static final String DEFAULT_FILE_PREFIX = "pgimport";
 	private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile";
 
@@ -35,6 +37,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 		File filePrefix;
 		boolean enableBboxBuilder;
 		boolean enableLinestringBuilder;
+		boolean enableKeepPartialLinestring;
 		NodeLocationStoreType storeType;
 
 		// Get the task arguments.
@@ -44,6 +47,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 				taskConfig, ARG_ENABLE_BBOX_BUILDER, DEFAULT_ENABLE_BBOX_BUILDER);
 		enableLinestringBuilder = getBooleanArgument(
 				taskConfig, ARG_ENABLE_LINESTRING_BUILDER, DEFAULT_ENABLE_LINESTRING_BUILDER);
+		enableKeepPartialLinestring = getBooleanArgument(
+				taskConfig, ARG_ENABLE_KEEP_PARTIAL_LINESTRING, DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING);
 		storeType = Enum.valueOf(
 				NodeLocationStoreType.class,
 				getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE));
@@ -53,7 +58,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 
 		return new SinkManager(
 			taskConfig.getId(),
-			new PostgreSqlDumpWriter(filePrefix, enableBboxBuilder, enableLinestringBuilder, storeType),
+			new PostgreSqlDumpWriter(filePrefix, enableBboxBuilder, enableLinestringBuilder,
+				enableKeepPartialLinestring, storeType),
 			taskConfig.getPipeArgs()
 		);
 	}

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriter.java

@@ -70,6 +70,7 @@ public class PostgreSqlWriter implements Sink, EntityProcessor {
 	private DatabaseContext dbCtx;
 	private boolean enableBboxBuilder;
 	private boolean enableLinestringBuilder;
+	private boolean enableKeepPartialLinestring;
 	private SchemaVersionValidator schemaVersionValidator;
 	private IndexManager indexManager;
 	private List<Node> nodeBuffer;
@@ -129,16 +130,21 @@ public class PostgreSqlWriter implements Sink, EntityProcessor {
 	 *            processing instead of relying on the database to build them
 	 *            after import. This increases processing but is faster than
 	 *            relying on the database.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 * @param storeType
 	 *            The node location storage type used by the geometry builders.
 	 */
 	public PostgreSqlWriter(
 			DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences,
-			boolean enableBboxBuilder, boolean enableLinestringBuilder, NodeLocationStoreType storeType) {
+			boolean enableBboxBuilder, boolean enableLinestringBuilder,
+			boolean enableKeepPartialLinestring, NodeLocationStoreType storeType) {
 		dbCtx = new DatabaseContext(loginCredentials);
 
 		this.enableBboxBuilder = enableBboxBuilder;
 		this.enableLinestringBuilder = enableLinestringBuilder;
+		this.enableKeepPartialLinestring = enableKeepPartialLinestring;
 
 		schemaVersionValidator = new SchemaVersionValidator(dbCtx, preferences);
 		indexManager = new IndexManager(dbCtx, !enableBboxBuilder, !enableLinestringBuilder);
@@ -387,7 +393,7 @@ private void flushWays(boolean complete) {
 					geometries.add(wayGeometryBuilder.createWayBbox(way));
 				}
 				if (enableLinestringBuilder) {
-					geometries.add(wayGeometryBuilder.createWayLinestring(way));
+					geometries.add(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring));
 				}
 				prmIndex = wayBuilder.populateEntityParameters(bulkWayStatement, prmIndex, way, geometries);
 			}
@@ -416,7 +422,7 @@ private void flushWays(boolean complete) {
 					geometries.add(wayGeometryBuilder.createWayBbox(way));
 				}
 				if (enableLinestringBuilder) {
-					geometries.add(wayGeometryBuilder.createWayLinestring(way));
+					geometries.add(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring));
 				}
 				wayBuilder.populateEntityParameters(singleWayStatement, 1, way, geometries);
 

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriterFactory.java

@@ -18,9 +18,11 @@
 public class PostgreSqlWriterFactory extends DatabaseTaskManagerFactory {
 	private static final String ARG_ENABLE_BBOX_BUILDER = "enableBboxBuilder";
 	private static final String ARG_ENABLE_LINESTRING_BUILDER = "enableLinestringBuilder";
+	private static final String ARG_ENABLE_KEEP_PARTIAL_LINESTRING = "enableKeepPartialLinestring";
 	private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType";
 	private static final boolean DEFAULT_ENABLE_BBOX_BUILDER = false;
 	private static final boolean DEFAULT_ENABLE_LINESTRING_BUILDER = false;
+	private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING = false;
 	private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile";
 
 	/**
@@ -32,6 +34,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 		DatabasePreferences preferences;
 		boolean enableBboxBuilder;
 		boolean enableLinestringBuilder;
+		boolean enableKeepPartialLinestring;
 		NodeLocationStoreType storeType;
 
 		// Get the task arguments.
@@ -40,13 +43,16 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
 		enableBboxBuilder = getBooleanArgument(taskConfig, ARG_ENABLE_BBOX_BUILDER, DEFAULT_ENABLE_BBOX_BUILDER);
 		enableLinestringBuilder = getBooleanArgument(
 				taskConfig, ARG_ENABLE_LINESTRING_BUILDER, DEFAULT_ENABLE_LINESTRING_BUILDER);
+		enableKeepPartialLinestring = getBooleanArgument(
+				taskConfig, ARG_ENABLE_KEEP_PARTIAL_LINESTRING, DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING);
 		storeType = Enum.valueOf(
 				NodeLocationStoreType.class,
 				getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE));
 
 		return new SinkManager(
 			taskConfig.getId(),
-			new PostgreSqlWriter(loginCredentials, preferences, enableBboxBuilder, enableLinestringBuilder, storeType),
+			new PostgreSqlWriter(loginCredentials, preferences, enableBboxBuilder, enableLinestringBuilder,
+				enableKeepPartialLinestring, storeType),
 			taskConfig.getPipeArgs()
 		);
 	}

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/CopyFilesetBuilder.java

@@ -35,6 +35,7 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor {
 
 	private boolean enableBboxBuilder;
 	private boolean enableLinestringBuilder;
+	private boolean enableKeepPartialLinestring;
 	private WayGeometryBuilder wayGeometryBuilder;
 	private CompletableContainer writerContainer;
 	private MemberTypeValueMapper memberTypeValueMapper;
@@ -66,14 +67,19 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor {
 	 *            processing instead of relying on the database to build them
 	 *            after import. This increases processing but is faster than
 	 *            relying on the database.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 * @param storeType
 	 *            The node location storage type used by the geometry builders.
 	 */
 	public CopyFilesetBuilder(
 			CopyFileset copyFileset, boolean enableBboxBuilder,
-			boolean enableLinestringBuilder, NodeLocationStoreType storeType) {
+			boolean enableLinestringBuilder, boolean enableKeepPartialLinestring,
+			NodeLocationStoreType storeType) {
 		this.enableBboxBuilder = enableBboxBuilder;
 		this.enableLinestringBuilder = enableLinestringBuilder;
+		this.enableKeepPartialLinestring = enableKeepPartialLinestring;
 
 		writerContainer = new CompletableContainer();
 
@@ -184,7 +190,7 @@ public void process(WayContainer wayContainer) {
 				wayWriter.writeField(wayGeometryBuilder.createWayBbox(way));
 			}
 			if (enableLinestringBuilder) {
-				wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way));
+				wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring));
 			}
 			wayWriter.endRecord();
 

osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/WayGeometryBuilder.java

@@ -191,9 +191,12 @@ public Polygon createWayBbox(Way way) {
 	 * 
 	 * @param way
 	 *            The way to create the linestring for.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 * @return The linestring representing the way.
 	 */
-	public LineString createWayLinestring(Way way) {
+	public LineString createWayLinestring(Way way, boolean enableKeepPartialLinestring) {
 		List<Point> linePoints;
 		int numValidNodes = 0;
 
@@ -207,7 +210,9 @@ public LineString createWayLinestring(Way way) {
 				numValidNodes++;
 				linePoints.add(new Point(nodeLocation.getLongitude(), nodeLocation.getLatitude()));
 			} else {
-				return null;
+				if (!enableKeepPartialLinestring) {
+					return null;
+				}
 			}
 		}
 

osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java

@@ -35,6 +35,7 @@ public class PostgreSqlCopyWriter implements Sink {
 	private NodeLocationStoreType storeType;
 	private boolean populateBbox;
 	private boolean populateLinestring;
+	private boolean enableKeepPartialLinestring;
 	private boolean keepInvalidWays;
 	private boolean initialized;
 	private DatabaseContext dbCtx;
@@ -49,17 +50,21 @@ public class PostgreSqlCopyWriter implements Sink {
 	 *            Contains preferences configuring database behaviour.
 	 * @param storeType
 	 *            The node location storage type used by the geometry builders.
+	 * @param enableKeepPartialLinestring
+	 *            If true, the way linestring is build even on invalid or missing
+	 *            nodes.
 	 * @param keepInvalidWays
 	 *            If true, zero and single node ways are kept. Otherwise they are
 	 *            silently dropped to avoid putting invalid geometries into the 
 	 *            database which can cause problems with postgis functions.
 	 */
 	public PostgreSqlCopyWriter(
 			DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences,
-			NodeLocationStoreType storeType, boolean keepInvalidWays) {
+			boolean enableKeepPartialLinestring, NodeLocationStoreType storeType, boolean keepInvalidWays) {
 		this.loginCredentials = loginCredentials;
 		this.preferences = preferences;
 		this.storeType = storeType;
+		this.enableKeepPartialLinestring = enableKeepPartialLinestring;
 		this.keepInvalidWays = keepInvalidWays;
 		this.dbCtx = new DatabaseContext(loginCredentials);
 		this.locker = new DatabaseLocker(dbCtx.getDataSource(), true);
@@ -76,7 +81,8 @@ private void initialize() {
 			populateLinestring = capabilityChecker.isWayLinestringSupported();
 
 			copyFilesetBuilder =
-				new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, storeType, keepInvalidWays);
+				new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, enableKeepPartialLinestring,
+					storeType, keepInvalidWays);
 
 			copyFilesetLoader = new CopyFilesetLoader(loginCredentials, preferences, copyFileset);