From 0b4ce770a87141dbc8c84abea40d6a424df1aa23 Mon Sep 17 00:00:00 2001 From: Samuel Thiriot Date: Tue, 19 Apr 2022 00:52:39 +0200 Subject: [PATCH] added a node to download from geofabrik related #74 --- .../.classpath | 5 + .../META-INF/MANIFEST.MF | 13 +- .../build.properties | 10 +- .../plugin.xml | 7 + .../pom.xml | 10 +- .../read_from_geofabrik/GeofabrikUtils.java | 169 ++++++ .../ReadWKTFromGeofabrikNodeDialog.java | 82 +++ .../ReadWKTFromGeofabrikNodeFactory.java | 73 +++ .../ReadWKTFromGeofabrikNodeFactory.xml | 39 ++ .../ReadWKTFromGeofabrikNodeModel.java | 536 ++++++++++++++++++ .../shp-wkt-reader-icon.png | Bin 0 -> 519 bytes .../reproject/ReprojectNodeModel.java | 69 +-- .../read_from_geofabrik/TestGeoFabrik.java | 23 + 13 files changed, 960 insertions(+), 76 deletions(-) create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/GeofabrikUtils.java create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeDialog.java create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.java create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.xml create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeModel.java create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/shp-wkt-reader-icon.png create mode 100644 ch.res_ear.samthiriot.knime.shapefilesaswkt/tests/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/TestGeoFabrik.java diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/.classpath b/ch.res_ear.samthiriot.knime.shapefilesaswkt/.classpath index c57822c..974f22c 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/.classpath +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/.classpath @@ -1,7 +1,11 @@ + + + + @@ -167,6 +171,7 @@ + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/META-INF/MANIFEST.MF b/ch.res_ear.samthiriot.knime.shapefilesaswkt/META-INF/MANIFEST.MF index ab091ba..42b1611 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/META-INF/MANIFEST.MF +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/META-INF/MANIFEST.MF @@ -158,7 +158,14 @@ Bundle-ClassPath: target/classes/, lib/txw2-2.4.0-b180830.0438.jar, lib/unit-api-2.0.jar, lib/uom-lib-common-2.0.jar, - lib/xml-commons-resolver-1.2.jar + lib/xml-commons-resolver-1.2.jar, + lib/gt-geojson-26.3.jar, + lib/hamcrest-core-1.1.jar, + lib/json-simple-1.1.1.jar, + lib/junit-4.10.jar, + lib/gt-geojsondatastore-26.3.jar, + lib/jackson-annotations-2.10.5.jar, + lib/jackson-databind-2.10.5.1.jar Bundle-Activator: ch.res_ear.samthiriot.knime.shapefilesaswkt.ShapefileAsWKTNodePlugin Bundle-Vendor: EIFER (European Institute for Energy Research) Require-Bundle: org.knime.workbench.core;bundle-version="[3.7.0,5.0.0)", @@ -166,7 +173,9 @@ Require-Bundle: org.knime.workbench.core;bundle-version="[3.7.0,5.0.0)", org.knime.product;bundle-version="[3.7.0,5.0.0)", org.knime.workbench;bundle-version="[3.7.0,5.0.0)", org.knime.testing;bundle-version="[3.7.0,5.0.0)", - org.eclipse.core.resources + org.eclipse.core.resources, + org.junit, + org.junit.jupiter.api Bundle-ActivationPolicy: lazy Export-Package: ., au.com.objectix.jgridshift, diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/build.properties b/ch.res_ear.samthiriot.knime.shapefilesaswkt/build.properties index 0511c45..117483b 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/build.properties +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/build.properties @@ -55,9 +55,8 @@ bin.includes = plugin.xml,\ src/ch/res_ear/samthiriot/knime/shapefilesaswkt/create/pointfrom2d/shapefilesAsWKT.png,\ src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/randompoint/RandomPointInShapeNodeFactory.xml,\ src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/randompoint/shapefilesAsWKT.png,\ - lib/,\ - lib/aircompressor-0.20.jar,\ - target/classes/ + target/classes/,\ + lib/ src.includes = icons/,\ pom.xml,\ src/**/*.xml,\ @@ -218,4 +217,7 @@ jars.extra.classpath = lib/aircompressor-0.20.jar,\ lib/txw2-2.4.0-b180830.0438.jar,\ lib/unit-api-2.0.jar,\ lib/uom-lib-common-2.0.jar,\ - lib/xml-commons-resolver-1.2.jar + lib/xml-commons-resolver-1.2.jar,\ + lib/gt-geojsondatastore-26.3.jar,\ + lib/jackson-annotations-2.10.5.jar,\ + lib/jackson-databind-2.10.5.1.jar diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/plugin.xml b/ch.res_ear.samthiriot.knime.shapefilesaswkt/plugin.xml index af58ea7..b912f66 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/plugin.xml +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/plugin.xml @@ -64,6 +64,13 @@ category-path="/community/spatialAsWKT/readSpatialAsWKT" factory-class="ch.res_ear.samthiriot.knime.shapefilesaswkt.read.read_from_gml.ReadGMLAsWKTNodeFactory" id="ch.res_ear.samthiriot.knime.shapefilesAsWKT.readFromGML.ReadGMLAsWKTNodeFactory"> + + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/pom.xml b/ch.res_ear.samthiriot.knime.shapefilesaswkt/pom.xml index dce9864..c38422a 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/pom.xml +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/pom.xml @@ -87,7 +87,13 @@ gt-image ${geotools.version} - + + + org.geotools + gt-geojsondatastore + ${geotools.version} + + org.geotools.jdbc gt-jdbc-postgis @@ -155,7 +161,7 @@ gt-xsd-gml3 ${geotools.version} - + com.google.maps diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/GeofabrikUtils.java b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/GeofabrikUtils.java new file mode 100644 index 0000000..be92853 --- /dev/null +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/GeofabrikUtils.java @@ -0,0 +1,169 @@ +package ch.res_ear.samthiriot.knime.shapefilesaswkt.read.read_from_geofabrik; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.io.IOUtils; +import org.geotools.data.FeatureReader; +import org.geotools.data.FileDataStore; +import org.geotools.data.geojson.GeoJSONDataStoreFactory; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.geotools.data.simple.SimpleFeatureIterator; +import org.geotools.data.simple.SimpleFeatureSource; +import org.opengis.feature.simple.SimpleFeature; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class GeofabrikUtils { + + public static final String URL_INDEX_GEOFABRIK = "https://download.geofabrik.de/index-v1.json"; + // + +// https://download.geofabrik.de/index-v1-nogeom.json + + public static File readGeofabrikIndexIntoFile() { + + + try { + + // load the file first + URL url = new URL(URL_INDEX_GEOFABRIK); + InputStream inputStream = url.openStream(); + + File f = new File("/tmp/test.json"); + if (f.exists()) + return f; + + FileWriter fileWriter = new FileWriter(f); + + IOUtils.copy(inputStream, fileWriter, StandardCharsets.UTF_8); + + return f; + + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException("unable to read the file"); + } + + } + + private static Map cachedName2Url = null; + + public static Map fetchListOfDataExtracts() { + + if (cachedName2Url != null) + return cachedName2Url; + + GeoJSONDataStoreFactory factory = new GeoJSONDataStoreFactory(); + + FileDataStore store = null; + SimpleFeatureIterator it = null; + try { + + /*Map params = new HashMap<>(); + params.put(GeoJSONDataStoreFactory.URL_PARAM.key, new URL(URL_INDEX_GEOFABRIK)); +*/ + + File f = readGeofabrikIndexIntoFile(); + + store = factory.createDataStore(f); + + SimpleFeatureCollection features = store.getFeatureSource(store.getTypeNames()[0]).getFeatures(); + + it = features.features(); + + Map name2url = new HashMap<>(); + + Map id2parent = new HashMap<>(); + Map id2name = new HashMap(); + + // decode features + while (it.hasNext()) { + SimpleFeature ft = it.next(); + String id = ft.getAttribute("id").toString(); + String name = ft.getAttribute("name").toString(); + + id2name.put(id, name); + + // find parent (optional) + String parent = (String) ft.getAttribute("parent"); + if (parent != null) { + id2parent.put(id, parent); + //System.out.println(id+" has for parent "+parent); + } else { + //System.out.println(id+" has no parent"); + } + + // find url + ObjectNode urls = (ObjectNode)ft.getAttribute("urls"); + JsonNode nodeShp = urls.findValue("shp"); + // skip index without shp + if (nodeShp == null) + continue; + + String shp = nodeShp.asText(); + + name2url.put(id, shp); + + /* + System.out.println(ft); + System.out.println(id); + System.out.println(name); + System.out.println(shp); + + System.out.println(); + */ + } + + // find parents + Map hierarchy2url = new TreeMap<>(); + for (String id: name2url.keySet()) { + + final String url = name2url.get(id); + + String newId = id2name.get(id); + + String parent = id2parent.get(id); + while (parent != null) { + newId = id2name.get(parent) +"/" + newId; + id = parent; + parent = id2parent.get(id); + } + + hierarchy2url.put(newId, url); + } + + cachedName2Url = Collections.unmodifiableMap(hierarchy2url); + return cachedName2Url; + + } catch (MalformedURLException e) { + e.printStackTrace(); + throw new RuntimeException("The index URL seems corrupted: "+URL_INDEX_GEOFABRIK, e); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("unable to access the list of data from "+URL_INDEX_GEOFABRIK, e); + } finally { + if (it != null) + it.close(); + + if (store != null) + store.dispose(); + + } + + + } +} diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeDialog.java b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeDialog.java new file mode 100644 index 0000000..cc73604 --- /dev/null +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeDialog.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2019 EIfER[1] (European Institute for Energy Research). + * This program and the accompanying materials + * are made available under the terms of the GNU GENERAL PUBLIC LICENSE + * which accompanies this distribution, and is available at + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Contributors: + * Samuel Thiriot - original version and contributions + *******************************************************************************/ +package ch.res_ear.samthiriot.knime.shapefilesaswkt.read.read_from_geofabrik; + +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; + +import org.knime.core.node.defaultnodesettings.DefaultNodeSettingsPane; +import org.knime.core.node.defaultnodesettings.DialogComponentStringSelection; +import org.knime.core.node.defaultnodesettings.SettingsModelString; + +/** + * This is an example implementation of the node dialog of the + * "ReadWKTFromDatabase" node. + * + * This node dialog derives from {@link DefaultNodeSettingsPane} which allows + * creation of a simple dialog with standard components. If you need a more + * complex dialog please derive directly from + * {@link org.knime.core.node.NodeDialogPane}. In general, one can create an + * arbitrary complex dialog using Java Swing. + * + * @author Samuel Thiriot + */ +public class ReadWKTFromGeofabrikNodeDialog extends DefaultNodeSettingsPane { + + + /** + * New dialog pane for configuring the node. The dialog created here + * will show up when double clicking on a node in KNIME Analytics Platform. + */ + protected ReadWKTFromGeofabrikNodeDialog() { + super(); + + { + SettingsModelString m_nameToLoad = new SettingsModelString("name_to_load", "Europe/Germany/Baden-Württemberg/Regierungsbezirk Karlsruhe"); + + Collection names = GeofabrikUtils.fetchListOfDataExtracts().keySet(); + + addDialogComponent(new DialogComponentStringSelection( + m_nameToLoad, + "Zone to load", + names + )); + } + + { + SettingsModelString m_layerToLoad = new SettingsModelString( + "layer_to_load", + "buildings (Building outlines)"); + + Collection layers = new LinkedList<>(Arrays.asList( + "buildings (Building outlines)", + "landuse (Forests, residential areas, industrial areas,...)", + "natural (Natural features)", + "places (Cities, towns, suburbs, villages,...)", + "pofw (Places of worship such as churches, mosques, ...)", + "pois (Points of Interest)", + "railways (Railway, subways, light rail, trams, ...)", + "roads (Roads, tracks, paths, ...)", + "traffic", + "transport (Parking lots, petrol (gas) stations, ...)", + "water (Lakes, ...)", + "waterways (Rivers, canals, streams, ...)" + )); + addDialogComponent(new DialogComponentStringSelection( + m_layerToLoad, + "layer to load", + layers + )); + } + } +} + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.java b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.java new file mode 100644 index 0000000..f9e05c7 --- /dev/null +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2019 EIfER[1] (European Institute for Energy Research). + * This program and the accompanying materials + * are made available under the terms of the GNU GENERAL PUBLIC LICENSE + * which accompanies this distribution, and is available at + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Contributors: + * Samuel Thiriot - original version and contributions + *******************************************************************************/ +package ch.res_ear.samthiriot.knime.shapefilesaswkt.read.read_from_geofabrik; + +import org.knime.core.node.NodeDialogPane; +import org.knime.core.node.NodeFactory; +import org.knime.core.node.NodeView; + +/** + * This is an example implementation of the node factory of the + * "ReadWKTFromDatabase" node. + * + * @author Samuel Thiriot + */ +public class ReadWKTFromGeofabrikNodeFactory + extends NodeFactory { + + /** + * {@inheritDoc} + */ + @Override + public ReadWKTFromGeofabrikNodeModel createNodeModel() { + // Create and return a new node model. + return new ReadWKTFromGeofabrikNodeModel(); + } + + /** + * {@inheritDoc} + */ + @Override + public int getNrNodeViews() { + // The number of views the node should have, in this cases there is none. + return 0; + } + + /** + * {@inheritDoc} + */ + @Override + public NodeView createNodeView(final int viewIndex, + final ReadWKTFromGeofabrikNodeModel nodeModel) { + // We return null as this example node does not provide a view. Also see "getNrNodeViews()". + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean hasDialog() { + // Indication whether the node has a dialog or not. + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public NodeDialogPane createNodeDialogPane() { + // This example node has a dialog, hence we create and return it here. Also see "hasDialog()". + return new ReadWKTFromGeofabrikNodeDialog(); + } + +} + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.xml b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.xml new file mode 100644 index 0000000..6549206 --- /dev/null +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeFactory.xml @@ -0,0 +1,39 @@ + + + + + Load Geometries From Geofabrik + + + Load Geometries from Geofabrik. + + + + + Load geometries from Geofabrik download. + Geofabrik + You can read their full documentation here. + +

+ The OpenStreetMap (OSM) project (www.openstreetmap.org) has collected an enormous + amount of free spatial data and the database is growing every day. Many people want to use + this data for their own GIS projects but have been hindered by the use of a non-standard + data format in the OSM project. +

+ + + + + + + + + Spatial population + + + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeModel.java b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeModel.java new file mode 100644 index 0000000..6408f5d --- /dev/null +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/ReadWKTFromGeofabrikNodeModel.java @@ -0,0 +1,536 @@ +/******************************************************************************* + * Copyright (c) 2019 EIfER[1] (European Institute for Energy Research). + * This program and the accompanying materials + * are made available under the terms of the GNU GENERAL PUBLIC LICENSE + * which accompanies this distribution, and is available at + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Contributors: + * Samuel Thiriot - original version and contributions + *******************************************************************************/ +package ch.res_ear.samthiriot.knime.shapefilesaswkt.read.read_from_geofabrik; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.CountingOutputStream; +import org.geotools.data.DataStore; +import org.geotools.data.DataStoreFinder; +import org.geotools.data.shapefile.ShapefileDataStore; +import org.geotools.data.simple.SimpleFeatureIterator; +import org.knime.core.data.DataCell; +import org.knime.core.data.DataColumnProperties; +import org.knime.core.data.DataColumnSpec; +import org.knime.core.data.DataColumnSpecCreator; +import org.knime.core.data.DataTableSpec; +import org.knime.core.data.MissingCell; +import org.knime.core.data.RowKey; +import org.knime.core.data.def.BooleanCell; +import org.knime.core.data.def.BooleanCell.BooleanCellFactory; +import org.knime.core.data.def.DefaultRow; +import org.knime.core.data.def.IntCell; +import org.knime.core.data.def.IntCell.IntCellFactory; +import org.knime.core.data.def.StringCell; +import org.knime.core.data.def.StringCell.StringCellFactory; +import org.knime.core.node.BufferedDataContainer; +import org.knime.core.node.BufferedDataTable; +import org.knime.core.node.CanceledExecutionException; +import org.knime.core.node.ExecutionContext; +import org.knime.core.node.ExecutionMonitor; +import org.knime.core.node.InvalidSettingsException; +import org.knime.core.node.NodeModel; +import org.knime.core.node.NodeSettingsRO; +import org.knime.core.node.NodeSettingsWO; +import org.knime.core.node.defaultnodesettings.SettingsModelString; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.referencing.crs.CoordinateReferenceSystem; + +import ch.res_ear.samthiriot.knime.shapefilesaswkt.SpatialUtils; + + +/** + * This is an example implementation of the node model of the + * "ReadWKTFromDatabase" node. + * + * This example node performs simple number formatting + * ({@link String#format(String, Object...)}) using a user defined format string + * on all double columns of its input table. + * + * @author Samuel Thiriot + */ +public class ReadWKTFromGeofabrikNodeModel extends NodeModel { + + SettingsModelString m_nameToLoad = new SettingsModelString( + "name_to_load", + "Europe/Germany/Baden-Württemberg/Regierungsbezirk Karlsruhe"); + + SettingsModelString m_layerToLoad = new SettingsModelString( + "layer_to_load", + "buildings (Building outlines)"); + + MissingCell missing = new MissingCell("not provided"); + + protected ReadWKTFromGeofabrikNodeModel() { + + super(0, 1); + + } + + + private DataColumnSpec[] createSpecs(CoordinateReferenceSystem coordinateReferenceSystem) { + + List specs = new LinkedList(); + + + // based on http://download.geofabrik.de/osm-data-in-gis-formats-free.pdf + + // standard specs + + specs.add(new DataColumnSpecCreator( + "id", + StringCell.TYPE + ).createSpec()); + + { + DataColumnSpecCreator creator = new DataColumnSpecCreator( + SpatialUtils.GEOMETRY_COLUMN_NAME, + StringCell.TYPE + ); + if (coordinateReferenceSystem != null) { + Map properties = new HashMap(); + coordinateReferenceSystem = SpatialUtils.getCRSforString("EPSG:4326"); + properties.put(SpatialUtils.PROPERTY_CRS_CODE, SpatialUtils.getStringForCRS(coordinateReferenceSystem)); + properties.put(SpatialUtils.PROPERTY_CRS_WKT, coordinateReferenceSystem.toWKT()); + DataColumnProperties propertiesKWT = new DataColumnProperties(properties); + creator.setProperties(propertiesKWT); + } + specs.add(creator.createSpec()); + } + specs.add(new DataColumnSpecCreator( + "osm_id", + StringCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "code", + IntCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "fclass", + StringCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "name", + StringCell.TYPE + ).createSpec()); + + if (m_layerToLoad.getStringValue().startsWith("places")) { + specs.add(new DataColumnSpecCreator( + "population", + IntCell.TYPE + ).createSpec()); + } else if (m_layerToLoad.getStringValue().startsWith("roads")) { + specs.add(new DataColumnSpecCreator( + "ref", + StringCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "oneway", + StringCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "maxspeed", + IntCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "layer", + IntCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "bridge", + BooleanCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "tunnel", + BooleanCell.TYPE + ).createSpec()); + } else if (m_layerToLoad.getStringValue().startsWith("railways")) { + specs.add(new DataColumnSpecCreator( + "layer", + IntCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "bridge", + BooleanCell.TYPE + ).createSpec()); + specs.add(new DataColumnSpecCreator( + "tunnel", + BooleanCell.TYPE + ).createSpec()); + } else if (m_layerToLoad.getStringValue().startsWith("waterways")) { + specs.add(new DataColumnSpecCreator( + "width", + IntCell.TYPE + ).createSpec()); + } else if (m_layerToLoad.getStringValue().startsWith("buildings")) { + specs.add(new DataColumnSpecCreator( + "type", + StringCell.TYPE + ).createSpec()); + } + + return specs.toArray(new DataColumnSpec[specs.size()]); + } + + + public static class DownloadCountingOutputStream extends CountingOutputStream { + + private final ExecutionContext exec; + private final double total; + private double done = 0; + + public DownloadCountingOutputStream( + OutputStream out, + ExecutionContext exec, + int total + ) { + super(out); + this.exec = exec; + this.total = total; + } + + @Override + protected void afterWrite(int n) throws IOException { + super.afterWrite(n); + + try { + exec.checkCanceled(); + } catch (CanceledExecutionException e) { + throw new RuntimeException("execution canceled"); + } + done += n; + //System.out.println(done+" / "+total+" "+(done/total)); + exec.setProgress(done/total); + } + + } + + protected DataCell getStringOrMissing(Object stringValue) { + if (stringValue == null) + return missing; + String str = stringValue.toString(); + if (str.isEmpty()) + return missing; + return StringCellFactory.create(str); + } + + protected DataCell getIntOrMissing(Object intValue) { + if (intValue == null) + return missing; + + return IntCellFactory.create(intValue.toString()); + } + + protected DataCell getIntOrMissingIfZero(Object intValue) { + if (intValue == null) + return missing; + + String str = intValue.toString(); + + if (str.equals("0")) + return missing; + + return IntCellFactory.create(str); + } + + protected DataCell getBoolOrMissing(Object val) { + if (val == null) + return missing; + + String str = val.toString(); + if (str.equals("T")) + return BooleanCellFactory.create(true); + else if (str.equals("F")) + return BooleanCellFactory.create(false); + else + return missing; + } + + @Override + protected BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception { + + + // download the file + String nameToDownload = m_nameToLoad.getStringValue(); + + exec.checkCanceled(); + exec.setMessage("finding URL..."); + + String urlToDownload = GeofabrikUtils.fetchListOfDataExtracts().get(nameToDownload); + URL urlToDownload2 = new URL(urlToDownload); + + exec.checkCanceled(); + exec.setMessage("estimating size"); + int total = Integer.parseInt(urlToDownload2.openConnection().getHeaderField("Content-Length")); + + exec.checkCanceled(); + exec.setMessage("downloading ("+(total/1024/1024)+" Mb)"); + File destFile = null; + { + ExecutionContext execCopy = exec.createSubExecutionContext(0.6); + destFile = File.createTempFile("geofabrik_", ".shp.zip"); + // TODO for debug + //destFile = new File("/tmp/toto.shp.zip"); + //if (!destFile.exists()) { + FileOutputStream os = new FileOutputStream(destFile); + DownloadCountingOutputStream cos = new DownloadCountingOutputStream(os, execCopy, total); + URLConnection connection = urlToDownload2.openConnection(); + connection.setUseCaches(true); + IOUtils.copy(connection.getInputStream(), cos); + cos.close(); + //} + } + + exec.checkCanceled(); + exec.setProgress(0.6, "unzipping"); + File destDir = Files.createTempDirectory("geofabrik").toFile(); + File fileForShp = null; + final String paramLayerToRead = m_layerToLoad.getStringValue(); + { + // find the code we search + String layerCode = paramLayerToRead; + int idxSpace = layerCode.indexOf(" "); + if (idxSpace > 0) + layerCode = layerCode.substring(0, idxSpace); + final String layerCodeToSearch = layerCode; + ZipFile zipFile = new ZipFile(destFile); + try { + Collection relevantEntries = zipFile + .stream() + .filter(entry -> entry.getName().contains(layerCodeToSearch)) + .collect(Collectors.toList()); + + for (ZipEntry zipEntry: relevantEntries) { + exec.setMessage("unzipping "+zipEntry.getName()); + InputStream is = zipFile.getInputStream(zipEntry); + File destFileEntry = new File(destDir, zipEntry.getName()); + FileOutputStream os = new FileOutputStream(destFileEntry); + IOUtils.copy(is, os); + if (zipEntry.getName().endsWith(".shp")) + fileForShp = destFileEntry; + } + + } finally { + zipFile.close(); + } + + } + + if (fileForShp == null) + throw new RuntimeException("no shp file found for layer "+paramLayerToRead+" for country "+nameToDownload); + + // open data using the good old shapefile reader + exec.checkCanceled(); + exec.setProgress(0.6, "loading"); + DataStore datastore = null; + { + + Map parameters = new HashMap<>(); + try { + parameters.put("url", fileForShp.toURI().toURL()); + } catch (MalformedURLException e2) { + throw new RuntimeException("cannot convert the path "+fileForShp+" to an URL", e2); + } + try { + getLogger().info("opening as a shapefile: "+fileForShp); + + datastore = DataStoreFinder.getDataStore(parameters); + } catch (IOException e1) { + e1.printStackTrace(); + throw new InvalidSettingsException("Unable to open the url as a shape file: "+e1.getMessage()); + } + + if (datastore == null) + throw new InvalidSettingsException("unable to open the shapefile from path "+fileForShp); + + // set the charset + try { + ((ShapefileDataStore)datastore).setCharset(StandardCharsets.UTF_8); + } catch (ClassCastException e) { + throw new InvalidSettingsException("unable to define charset for this datastore"); + } + + } + final String schemaName = datastore.getTypeNames()[0]; + + SimpleFeatureType type; + try { + type = datastore.getSchema(schemaName); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Unable to decode the schema "+schemaName+" from the file: "+e, e); + } + + // create result table + DataColumnSpec[] colSpecs = createSpecs(type.getCoordinateReferenceSystem()); + + DataTableSpec outputSpec = new DataTableSpec(colSpecs); + final BufferedDataContainer container = exec.createDataContainer(outputSpec); + + + int totalFeatures = datastore.getFeatureSource(schemaName).getFeatures().size(); + ExecutionContext execCreate = exec.createSubExecutionContext(0.4); + + SimpleFeatureIterator itFeature = datastore + .getFeatureSource(schemaName) + .getFeatures() + .features(); + int rowIdx = 0; + + + while (itFeature.hasNext()) { + SimpleFeature feature = itFeature.next(); + + int i=0; + DataCell[] cells = new DataCell[colSpecs.length]; + + cells[i++] = StringCellFactory.create(feature.getID()); + cells[i++] = StringCellFactory.create(feature.getAttribute("the_geom").toString()); + cells[i++] = StringCellFactory.create(feature.getAttribute("osm_id").toString()); + cells[i++] = IntCellFactory.create(feature.getAttribute("code").toString()); + cells[i++] = getStringOrMissing(feature.getAttribute("fclass")); + cells[i++] = getStringOrMissing(feature.getAttribute("name")); + + if (paramLayerToRead.startsWith("places")) { + cells[i++] = getIntOrMissing(feature.getAttribute("population")); + + } else if (paramLayerToRead.startsWith("roads")) { + cells[i++] = getStringOrMissing(feature.getAttribute("ref")); + cells[i++] = getStringOrMissing(feature.getAttribute("oneway").toString()); + cells[i++] = getIntOrMissingIfZero(feature.getAttribute("maxspeed")); + cells[i++] = IntCellFactory.create(feature.getAttribute("layer").toString()); + cells[i++] = getBoolOrMissing(feature.getAttribute("bridge")); + cells[i++] = getBoolOrMissing(feature.getAttribute("tunnel")); + + } else if (m_layerToLoad.getStringValue().startsWith("railways")) { + cells[i++] = IntCellFactory.create(feature.getAttribute("layer").toString()); + cells[i++] = getBoolOrMissing(feature.getAttribute("bridge").toString()); + cells[i++] = getBoolOrMissing(feature.getAttribute("tunnel").toString()); + + } else if (m_layerToLoad.getStringValue().startsWith("waterways")) { + cells[i++] = getIntOrMissingIfZero(feature.getAttribute("width")); + + } else if (m_layerToLoad.getStringValue().startsWith("buildings")) { + + cells[i++] = getStringOrMissing(feature.getAttribute("type")); + } + + container.addRowToTable( + new DefaultRow( + new RowKey("Row " + rowIdx), + cells + )); + + if (rowIdx % 10 == 0) { + // check if the execution monitor was canceled + exec.checkCanceled(); + execCreate.setProgress( + (double)rowIdx / totalFeatures, + "reading row " + rowIdx); + } + rowIdx++; + } + itFeature.close(); + + // clear data + // TODO + // if (destFile != null) + //destFile.delete(); + + datastore.dispose(); + + // once we are done, we close the container and return its table + container.close(); + BufferedDataTable out = container.getTable(); + return new BufferedDataTable[]{ out }; + + } + + + @Override + protected DataTableSpec[] configure(final DataTableSpec[] inSpecs) + throws InvalidSettingsException { + + return new DataTableSpec[]{ new DataTableSpec(createSpecs(null)) }; + } + + /** + * {@inheritDoc} + */ + @Override + protected void saveSettingsTo(final NodeSettingsWO settings) { + + m_nameToLoad.saveSettingsTo(settings); + m_layerToLoad.saveSettingsTo(settings); + } + + /** + * {@inheritDoc} + */ + @Override + protected void loadValidatedSettingsFrom(final NodeSettingsRO settings) throws InvalidSettingsException { + + m_nameToLoad.loadSettingsFrom(settings); + m_layerToLoad.loadSettingsFrom(settings); + + } + + /** + * {@inheritDoc} + */ + @Override + protected void validateSettings(final NodeSettingsRO settings) throws InvalidSettingsException { + + m_nameToLoad.validateSettings(settings); + m_layerToLoad.validateSettings(settings); + } + + @Override + protected void loadInternals(File nodeInternDir, ExecutionMonitor exec) + throws IOException, CanceledExecutionException { + + // nothing to do + } + + @Override + protected void saveInternals(File nodeInternDir, ExecutionMonitor exec) + throws IOException, CanceledExecutionException { + + // nothing to do + } + + @Override + protected void reset() { + + // nothing to do + } + +} + diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/shp-wkt-reader-icon.png b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/read/read_from_geofabrik/shp-wkt-reader-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..b3d02e3e501f9f22e2007c623978ee54c1c5d0dd GIT binary patch literal 519 zcmV+i0{H!jP)Cmyz z(GTEeKSQUwi)%;m0~Csrn+~Gsq9RgaxGfHrBu(Of;l}qk+#rd$7xh`r;o&^LbB@#+ z)j(09Ltdg^JQvlVp^8Cq6hOpa+>|jQj&WZA;C@?Kba|lsRGv>0*9=zX2{cWg)WIat z$!&15oRx~4{@&s0>I4A?T40GX!@Ix+rxV6)x-0&e2~0<1^) zX<)kD9|7~gMijsWzDM}EHsBgq1&VPO#OWjO6L|C7VCQ8|(jCGRh9&UmT1)@{002ov JPDHLkV1hzg+qwV% literal 0 HcmV?d00001 diff --git a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/reproject/ReprojectNodeModel.java b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/reproject/ReprojectNodeModel.java index 4a9cfa2..76ed1d7 100644 --- a/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/reproject/ReprojectNodeModel.java +++ b/ch.res_ear.samthiriot.knime.shapefilesaswkt/src/ch/res_ear/samthiriot/knime/shapefilesaswkt/transform/reproject/ReprojectNodeModel.java @@ -164,79 +164,12 @@ protected BufferedDataTable[] execute(final BufferedDataTable[] inData, if (done++ % 10 == 0) { exec.checkCanceled(); - exec.setProgress(done/total, "computing surface of row "+done); + exec.setProgress(done/total, "reprojecting row "+done); } } ); - /* - // copy the input population into a datastore - exec.setMessage("spatializing inputs"); - DataStore datastore = SpatialUtils.createDataStore(); - Runnable runnableSpatialize = SpatialUtils.decodeAsFeaturesRunnable( - inputPopulation, - SpatialUtils.GEOMETRY_COLUMN_NAME, - execProgressSpatialiseInputs, - datastore, - "entities", - crsOrig, - false, - null - ); - ExecutorService executor = Executors.newSingleThreadExecutor(); - executor.execute(runnableSpatialize); - executor.shutdown(); - try { - // wait forever - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - exec.setMessage("reprojecting"); - - Iterator itRow = inputPopulation.iterator(); - - // TODO use http://docs.geotools.org/latest/tutorials/geometry/geometrycrs.html - - SimpleFeatureCollection features = datastore.getFeatureSource(datastore.getNames().get(0)).getFeatures(); - - ReprojectingFeatureCollection rfc = new ReprojectingFeatureCollection( - features, crsTarget); - SimpleFeatureIterator itEntities = rfc.features(); - long currentIdx = 0; - long total = inputPopulation.size(); - while (itEntities.hasNext()) { - if (!itRow.hasNext()) throw new RuntimeException("program error: wrong size"); - - DataRow row = itRow.next(); - - SimpleFeature parentFeature = itEntities.next(); - Geometry geom = (Geometry)parentFeature.getDefaultGeometry(); - - - DataCell[] novelCells = new DataCell[row.getNumCells()]; - for (int i=0; i name2url = GeofabrikUtils.fetchListOfDataExtracts(); + + for (String name: name2url.keySet()) + System.out.println(name); + + + //fail("Not yet implemented"); + } + +}