Add an option to treat empty CSV columns as nulls, not empty strings

Author: e-n-f

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.30.1
+
+* Ensure that per-feature minzoom and maxzoom are integers
+* Report compression errors in tippecanoe-decode
+* Add the ability to specify the file format with -L{"format":"…"}
+* Add an option to treat empty CSV columns as nulls, not empty strings
+
 ## 1.30.0
 
 * Add a filter extension to allow filtering individual attributes

Makefile

@@ -178,20 +178,24 @@ join-test: tile-join
 	rm -f tests/join-population/macarthur-6-9.mbtiles.json.check tests/join-population/macarthur-6-9.mbtiles
 	./tippecanoe -q -f -d10 -D10 -Z9 -z11 -o tests/join-population/macarthur2.mbtiles -l macarthur tests/join-population/macarthur2.json
 	./tile-join --quiet --force -o tests/join-population/joined.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force -o tests/join-population/joined-null.mbtiles --empty-csv-columns-are-null -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join --quiet --force --no-tile-stats -o tests/join-population/joined-no-tile-stats.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -i -o tests/join-population/joined-i.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -o tests/join-population/merged.mbtiles tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
 	./tile-join -q -f -c tests/join-population/windows.csv -o tests/join-population/windows.mbtiles tests/join-population/macarthur.mbtiles
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined.mbtiles > tests/join-population/joined.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-null.mbtiles > tests/join-population/joined-null.mbtiles.json.check
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-no-tile-stats.mbtiles > tests/join-population/joined-no-tile-stats.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/joined-i.mbtiles > tests/join-population/joined-i.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/merged.mbtiles > tests/join-population/merged.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/windows.mbtiles > tests/join-population/windows.mbtiles.json.check
 	cmp tests/join-population/joined.mbtiles.json.check tests/join-population/joined.mbtiles.json
+	cmp tests/join-population/joined-null.mbtiles.json.check tests/join-population/joined-null.mbtiles.json
 	cmp tests/join-population/joined-no-tile-stats.mbtiles.json.check tests/join-population/joined-no-tile-stats.mbtiles.json
 	cmp tests/join-population/joined-i.mbtiles.json.check tests/join-population/joined-i.mbtiles.json
 	cmp tests/join-population/merged.mbtiles.json.check tests/join-population/merged.mbtiles.json
 	cmp tests/join-population/windows.mbtiles.json.check tests/join-population/windows.mbtiles.json
+	rm -f tests/join-population/joined-null.mbtiles tests/join-population/joined-null.mbtiles.json.check
 	./tile-join -q -f -l macarthur -n "macarthur name" -N "macarthur description" -A "macarthur attribution" -o tests/join-population/just-macarthur.mbtiles tests/join-population/merged.mbtiles
 	./tile-join -q -f -L macarthur -o tests/join-population/no-macarthur.mbtiles tests/join-population/merged.mbtiles
 	./tippecanoe-decode -x generator tests/join-population/just-macarthur.mbtiles > tests/join-population/just-macarthur.mbtiles.json.check
@@ -253,8 +257,11 @@ join-filter-test:
 json-tool-test: tippecanoe-json-tool
 	./tippecanoe-json-tool -e GEOID10 tests/join-population/tabblock_06001420.json | sort > tests/join-population/tabblock_06001420.json.sort
 	./tippecanoe-json-tool -c tests/join-population/population.csv tests/join-population/tabblock_06001420.json.sort > tests/join-population/tabblock_06001420.json.sort.joined
+	./tippecanoe-json-tool --empty-csv-columns-are-null -c tests/join-population/population.csv tests/join-population/tabblock_06001420.json.sort > tests/join-population/tabblock_06001420-null.json.sort.joined
 	cmp tests/join-population/tabblock_06001420.json.sort.joined tests/join-population/tabblock_06001420.json.sort.joined.standard
+	cmp tests/join-population/tabblock_06001420-null.json.sort.joined tests/join-population/tabblock_06001420-null.json.sort.joined.standard
 	rm -f tests/join-population/tabblock_06001420.json.sort tests/join-population/tabblock_06001420.json.sort.joined
+	rm -f tests/join-population/tabblock_06001420-null.json.sort.joined
 
 allow-existing-test:
 	# Make a tileset
@@ -283,6 +290,11 @@ csv-test:
 	./tippecanoe-decode -x generator tests/csv/out.mbtiles > tests/csv/out.mbtiles.json.check
 	cmp tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles.json
 	rm -f tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles
+	# Reading from named CSV, with nulls
+	./tippecanoe -q --empty-csv-columns-are-null -zg -f -o tests/csv/out-null.mbtiles tests/csv/ne_110m_populated_places_simple.csv
+	./tippecanoe-decode -x generator tests/csv/out-null.mbtiles > tests/csv/out-null.mbtiles.json.check
+	cmp tests/csv/out-null.mbtiles.json.check tests/csv/out-null.mbtiles.json
+	rm -f tests/csv/out-null.mbtiles.json.check tests/csv/out-null.mbtiles
 	# Same, but specifying csv with -L format
 	./tippecanoe -q -zg -f -o tests/csv/out.mbtiles -L'{"file":"", "format":"csv", "layer":"ne_110m_populated_places_simplecsv"}' < tests/csv/ne_110m_populated_places_simple.csv
 	./tippecanoe-decode -x generator tests/csv/out.mbtiles > tests/csv/out.mbtiles.json.check

README.md

@@ -214,6 +214,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_.
    that are dropped, coalesced-as-needed, or clustered. The _operation_ may be
    `sum`, `product`, `mean`, `max`, `min`, `concat`, or `comma`
    to specify how the named _attribute_ is accumulated onto the attribute of the same name in a feature that does survive.
+ * `-pe` or `--empty-csv-columns-are-null`: Treat empty CSV columns as nulls rather than as empty strings.
 
 ### Filtering features by attributes
 
@@ -561,6 +562,7 @@ The options are:
  * `-i` or `--if-matched`: Only include features that matched the CSV.
  * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://www.mapbox.com/mapbox-gl-js/style-spec/#types-filter)) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers.
  * `-J` *filter-file* or `--feature-filter-file`=*filter-file*: Like `-j`, but read the filter from a file.
+ * `-pe` or `--empty-csv-columns-are-null`: Treat empty CSV columns as nulls rather than as empty strings.
 
 ### Setting or disabling tile size limits
 
@@ -686,6 +688,7 @@ something better.
    The formatting makes excessive use of `\u` quoting so that it follows JSON string rules but will still
    be sorted correctly by tools that just do ASCII comparisons.
  * `-c` *file.csv* or `--csv=`*file.csv*: Join attributes from the named sorted CSV file, using its first column as the join key. Geometries will be passed through even if they do not match the CSV; CSV lines that do not match a geometry will be discarded.
+ * `-pe` or `--empty-csv-columns-are-null`: Treat empty CSV columns as nulls rather than as empty strings.
 
 ### Example
 

geocsv.cpp

@@ -8,6 +8,7 @@
 #include "text.hpp"
 #include "csv.hpp"
 #include "milo/dtoa_milo.h"
+#include "options.hpp"
 
 void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fname, int layer, std::string layername) {
 	FILE *f;
@@ -97,6 +98,9 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
 				serial_val sv;
 				if (is_number(line[i])) {
 					sv.type = mvt_double;
+				} else if (line[i].size() == 0 && prevent[P_EMPTY_CSV_COLUMNS]) {
+					sv.type = mvt_null;
+					line[i] = "null";
 				} else {
 					sv.type = mvt_string;
 				}

jsontool.cpp

@@ -18,6 +18,7 @@ const char *extract = NULL;
 FILE *csvfile = NULL;
 std::vector<std::string> header;
 std::vector<std::string> fields;
+int pe = false;
 
 std::string buffered;
 int buffered_type = -1;
@@ -318,9 +319,11 @@ void join_csv(json_object *j) {
 				} else if (is_number(v)) {
 					attr_type = JSON_NUMBER;
 				}
+			} else if (pe) {
+				attr_type = JSON_NULL;
 			}
 
-			{
+			if (attr_type != JSON_NULL) {
 				// This knows more about the structure of JSON objects than it ought to
 
 				json_object *ko = (json_object *) malloc(sizeof(json_object));
@@ -433,6 +436,8 @@ int main(int argc, char **argv) {
 		{"wrap", no_argument, 0, 'w'},
 		{"extract", required_argument, 0, 'e'},
 		{"csv", required_argument, 0, 'c'},
+		{"empty-csv-columns-are-null", no_argument, &pe, 1},
+		{"prevent", required_argument, 0, 'p'},
 
 		{0, 0, 0, 0},
 	};
@@ -453,6 +458,9 @@ int main(int argc, char **argv) {
 
 	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, NULL)) != -1) {
 		switch (i) {
+		case 0:
+			break;
+
 		case 'w':
 			wrap = true;
 			break;
@@ -465,6 +473,15 @@ int main(int argc, char **argv) {
 			csv = optarg;
 			break;
 
+		case 'p':
+			if (strcmp(optarg, "e") == 0) {
+				pe = true;
+			} else {
+				fprintf(stderr, "%s: Unknown option for -p%s\n", argv[0], optarg);
+				exit(EXIT_FAILURE);
+			}
+			break;
+
 		default:
 			fprintf(stderr, "Unexpected option -%c\n", i);
 			exit(EXIT_FAILURE);

main.cpp

@@ -2517,6 +2517,7 @@ int main(int argc, char **argv) {
 		{"attribute-type", required_argument, 0, 'T'},
 		{"attribute-description", required_argument, 0, 'Y'},
 		{"accumulate-attribute", required_argument, 0, 'E'},
+		{"empty-csv-columns-are-null", no_argument, &prevent[P_EMPTY_CSV_COLUMNS], 1},
 
 		{"Filtering features by attributes", 0, 0, 0},
 		{"feature-filter-file", required_argument, 0, 'J'},

man/tippecanoe.1

@@ -244,6 +244,8 @@ If the type is \fB\fCint\fR and the original attribute was floating\-point, it i
 that are dropped, coalesced\-as\-needed, or clustered. The \fIoperation\fP may be
 \fB\fCsum\fR, \fB\fCproduct\fR, \fB\fCmean\fR, \fB\fCmax\fR, \fB\fCmin\fR, \fB\fCconcat\fR, or \fB\fCcomma\fR
 to specify how the named \fIattribute\fP is accumulated onto the attribute of the same name in a feature that does survive.
+.IP \(bu 2
+\fB\fC\-pe\fR or \fB\fC\-\-empty\-csv\-columns\-are\-null\fR: Treat empty CSV columns as nulls rather than as empty strings.
 .RE
 .SS Filtering features by attributes
 .RS
@@ -675,6 +677,8 @@ The options are:
 \fB\fC\-j\fR \fIfilter\fP or \fB\fC\-\-feature\-filter\fR=\fIfilter\fP: Check features against a per\-layer filter (as defined in the Mapbox GL Style Specification \[la]https://www.mapbox.com/mapbox-gl-js/style-spec/#types-filter\[ra]) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer \fB\fC"*"\fR apply to all layers.
 .IP \(bu 2
 \fB\fC\-J\fR \fIfilter\-file\fP or \fB\fC\-\-feature\-filter\-file\fR=\fIfilter\-file\fP: Like \fB\fC\-j\fR, but read the filter from a file.
+.IP \(bu 2
+\fB\fC\-pe\fR or \fB\fC\-\-empty\-csv\-columns\-are\-null\fR: Treat empty CSV columns as nulls rather than as empty strings.
 .RE
 .SS Setting or disabling tile size limits
 .RS
@@ -829,6 +833,8 @@ The formatting makes excessive use of \fB\fC\\u\fR quoting so that it follows JS
 be sorted correctly by tools that just do ASCII comparisons.
 .IP \(bu 2
 \fB\fC\-c\fR \fIfile.csv\fP or \fB\fC\-\-csv=\fR\fIfile.csv\fP: Join attributes from the named sorted CSV file, using its first column as the join key. Geometries will be passed through even if they do not match the CSV; CSV lines that do not match a geometry will be discarded.
+.IP \(bu 2
+\fB\fC\-pe\fR or \fB\fC\-\-empty\-csv\-columns\-are\-null\fR: Treat empty CSV columns as nulls rather than as empty strings.
 .RE
 .SS Example
 .PP

options.hpp

@@ -37,6 +37,7 @@
 #define P_TILE_STATS ((int) 'g')
 #define P_USE_SOURCE_POLYGON_WINDING ((int) 'w')
 #define P_REVERSE_SOURCE_POLYGON_WINDING ((int) 'W')
+#define P_EMPTY_CSV_COLUMNS ((int) 'e')
 
 extern int prevent[256];
 extern int additional[256];