From 459aef11204453b482a8c095957e33030e9a821a Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Fri, 9 Aug 2024 16:49:05 -0700 Subject: [PATCH] Try adding --drop-sparsest-as-needed --- main.cpp | 2 ++ options.hpp | 2 ++ serial.cpp | 4 +++- tile.cpp | 63 ++++++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/main.cpp b/main.cpp index dad6a05d..d4642c3b 100644 --- a/main.cpp +++ b/main.cpp @@ -3113,9 +3113,11 @@ int main(int argc, char **argv) { {"drop-densest-as-needed", no_argument, &additional[A_DROP_DENSEST_AS_NEEDED], 1}, {"drop-fraction-as-needed", no_argument, &additional[A_DROP_FRACTION_AS_NEEDED], 1}, {"drop-smallest-as-needed", no_argument, &additional[A_DROP_SMALLEST_AS_NEEDED], 1}, + {"drop-sparsest-as-needed", no_argument, &additional[A_DROP_SPARSEST_AS_NEEDED], 1}, {"coalesce-densest-as-needed", no_argument, &additional[A_COALESCE_DENSEST_AS_NEEDED], 1}, {"coalesce-fraction-as-needed", no_argument, &additional[A_COALESCE_FRACTION_AS_NEEDED], 1}, {"coalesce-smallest-as-needed", no_argument, &additional[A_COALESCE_SMALLEST_AS_NEEDED], 1}, + {"coalesce-sparsest-as-needed", no_argument, &additional[A_COALESCE_SPARSEST_AS_NEEDED], 1}, {"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1}, {"cluster-densest-as-needed", no_argument, &additional[A_CLUSTER_DENSEST_AS_NEEDED], 1}, diff --git a/options.hpp b/options.hpp index 30ddc599..62e9cd1b 100644 --- a/options.hpp +++ b/options.hpp @@ -14,9 +14,11 @@ #define A_DROP_DENSEST_AS_NEEDED ((int) 's') #define A_DROP_FRACTION_AS_NEEDED ((int) 'd') #define A_DROP_SMALLEST_AS_NEEDED ((int) 'n') +#define A_DROP_SPARSEST_AS_NEEDED ((int) 's' & 0x1F) #define A_COALESCE_DENSEST_AS_NEEDED ((int) 'S') #define A_COALESCE_SMALLEST_AS_NEEDED ((int) 'N') #define A_COALESCE_FRACTION_AS_NEEDED ((int) 'D') +#define A_COALESCE_SPARSEST_AS_NEEDED ((int) 't' & 0x1F) #define A_GRID_LOW_ZOOMS ((int) 'L') #define A_DETECT_WRAPAROUND ((int) 'w') #define A_EXTEND_ZOOMS ((int) 'e') diff --git a/serial.cpp b/serial.cpp index b043c102..9afc7055 100644 --- a/serial.cpp +++ b/serial.cpp @@ -681,7 +681,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: // keep old behavior, which loses one bit of precision at the bottom midx = (sf.bbox[0] / 2 + sf.bbox[2] / 2) & ((1LL << 32) - 1); midy = (sf.bbox[1] / 2 + sf.bbox[3] / 2) & ((1LL << 32) - 1); - } else if ((additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED]) && sf.t == VT_POLYGON) { + } else if ((additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED]) && sf.t == VT_POLYGON) { // This probably should really apply to all polygons, // but I hate to change the feature sequence in all the // test fixtures again @@ -725,6 +725,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: if (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || + additional[A_DROP_SPARSEST_AS_NEEDED] || + additional[A_COALESCE_SPARSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED] || additional[A_CALCULATE_FEATURE_DENSITY] || additional[A_DROP_SMALLEST_AS_NEEDED] || diff --git a/tile.cpp b/tile.cpp index 3e322160..664780ed 100644 --- a/tile.cpp +++ b/tile.cpp @@ -772,6 +772,17 @@ static unsigned long long choose_mingap(std::vector &gaps, d return gaps[ix]; } +static unsigned long long choose_maxgap(std::vector &gaps, double f, unsigned long long existing_gap) { + std::stable_sort(gaps.begin(), gaps.end()); + + size_t ix = (gaps.size() - 1) * f; + while (ix > 0 && gaps[ix] == existing_gap) { + ix--; + } + + return gaps[ix]; +} + // This function is called to choose the new "extent" threshold to try when a tile exceeds the // tile size limit or feature limit and `--drop-smallest-as-needed` or `--coalesce-smallest-as-needed` // has been set. @@ -888,6 +899,8 @@ struct write_tile_args { size_t pass = 0; unsigned long long mingap = 0; unsigned long long mingap_out = 0; + unsigned long long maxgap = 0; + unsigned long long maxgap_out = 0; long long minextent = 0; long long minextent_out = 0; unsigned long long mindrop_sequence = 0; @@ -1547,9 +1560,10 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, unsigned long long maxgap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; + double maxgap_fraction = 1; double minextent_fraction = 1; double mindrop_sequence_fraction = 1; @@ -1864,9 +1878,9 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch can_stop_early = false; continue; } - } else if (additional[A_DROP_DENSEST_AS_NEEDED]) { + } else if (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_DROP_SPARSEST_AS_NEEDED]) { add_sample_to(gaps, sf.gap, gaps_increment, seq); - if (sf.gap < mingap) { + if (sf.gap < mingap || sf.gap > maxgap) { if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) { can_stop_early = false; continue; @@ -1895,9 +1909,9 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch drop_rest = true; continue; } - } else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) { + } else if (additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED]) { add_sample_to(gaps, sf.gap, gaps_increment, seq); - if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) { + if ((sf.gap < mingap || sf.gap > maxgap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) { coalesce_geometry(features[which_serial_feature], sf); features[which_serial_feature].coalesced = true; coalesced_area += sf.extent; @@ -2529,6 +2543,21 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch line_detail++; continue; } + } else if (maxgap > 0 && (additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED])) { + maxgap_fraction = maxgap_fraction * scaled_max_tile_features / totalsize * 0.80; + unsigned long long m = choose_maxgap(gaps, maxgap_fraction, maxgap); + if (m != maxgap) { + maxgap = m; + if (maxgap < arg->maxgap_out) { + arg->maxgap_out = maxgap; + arg->still_dropping = true; + } + if (!quiet) { + fprintf(stderr, "Going to try keeping the densest %0.2f%% of the features to make it fit\n", maxgap_fraction * 100.0); + } + line_detail++; + continue; + } } else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) { minextent_fraction = minextent_fraction * scaled_max_tile_features / totalsize * 0.75; long long m = choose_minextent(extents, minextent_fraction, minextent); @@ -2641,6 +2670,21 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch line_detail++; continue; } + } else if (maxgap > 0 && (additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED])) { + maxgap_fraction = maxgap_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.80; + unsigned long long m = choose_maxgap(gaps, maxgap_fraction, maxgap); + if (m != maxgap) { + maxgap = m; + if (maxgap < arg->maxgap_out) { + arg->maxgap_out = maxgap; + arg->still_dropping = true; + } + if (!quiet) { + fprintf(stderr, "Going to try keeping the densest %0.2f%% of the features to make it fit\n", maxgap_fraction * 100.0); + } + line_detail++; + continue; + } } else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) { minextent_fraction = minextent_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.75; long long m = choose_minextent(extents, minextent_fraction, minextent); @@ -2810,7 +2854,7 @@ exit(EXIT_IMPOSSIBLE); skip_tile(&dc, &geompos, arg->compressed); len = 1; } else { - len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); + len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->maxgap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); } if (pthread_mutex_lock(&var_lock) != 0) { @@ -2987,6 +3031,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: double zoom_gamma = gamma; unsigned long long zoom_mingap = 0; + unsigned long long zoom_maxgap = ULLONG_MAX; long long zoom_minextent = 0; unsigned long long zoom_mindrop_sequence = 0; size_t zoom_tile_size = 0; @@ -3020,7 +3065,9 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: args[thread].gamma = zoom_gamma; args[thread].gamma_out = zoom_gamma; args[thread].mingap = zoom_mingap; + args[thread].maxgap = zoom_maxgap; args[thread].mingap_out = zoom_mingap; + args[thread].maxgap_out = zoom_maxgap; args[thread].minextent = zoom_minextent; args[thread].minextent_out = zoom_minextent; args[thread].mindrop_sequence = zoom_mindrop_sequence; @@ -3103,6 +3150,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: zoom_mingap = args[thread].mingap_out; again = true; } + if (args[thread].maxgap_out < zoom_maxgap) { + zoom_maxgap = args[thread].maxgap_out; + again = true; + } if (args[thread].minextent_out > zoom_minextent) { zoom_minextent = args[thread].minextent_out; again = true;