|
| 1 | +-- This config example file is released into the Public Domain. |
| 2 | +-- |
| 3 | +-- This config demonstrates a "grouped linemerge" done entirely in SQL with |
| 4 | +-- osm2pgsql.run_sql() in process_gen(). It merges connected lines that share |
| 5 | +-- the same grouping columns into single lines, the equivalent of |
| 6 | +-- |
| 7 | +-- SELECT cols..., (ST_Dump(ST_LineMerge(ST_Collect(geom)))).geom |
| 8 | +-- FROM roads GROUP BY cols... |
| 9 | +-- |
| 10 | +-- built once and maintained incrementally on updates, using ordinary tile |
| 11 | +-- expiry as the "what changed" signal. A typical use is merging road segments |
| 12 | +-- that render identically (same name/ref/highway/layer) so labels and shields |
| 13 | +-- sit on the whole road instead of on each OSM way. |
| 14 | +-- |
| 15 | +-- NOTE THAT THE GENERALIZATION SUPPORT IS EXPERIMENTAL AND MIGHT CHANGE |
| 16 | +-- WITHOUT NOTICE! |
| 17 | +-- |
| 18 | +-- Workflow (same as any gen config -- run osm2pgsql-gen after osm2pgsql): |
| 19 | +-- Import: osm2pgsql -O flex -S grouped-linemerge.lua DATA.osm.pbf |
| 20 | +-- Build merged table: osm2pgsql-gen -S grouped-linemerge.lua |
| 21 | +-- Apply an update: osm2pgsql -a -O flex -S grouped-linemerge.lua CHANGES.osc.gz |
| 22 | +-- Update merged table: osm2pgsql-gen -a -S grouped-linemerge.lua |
| 23 | + |
| 24 | +-- --------------------------------------------------------------------------- |
| 25 | +-- Configuration |
| 26 | +-- --------------------------------------------------------------------------- |
| 27 | +local SRC = 'roads' -- source table (one row per OSM way) |
| 28 | +local DEST = 'roads_merged' -- destination table (the merged lines) |
| 29 | +local EXPIRE = 'exp_roads' -- tile-expire table (the change signal) |
| 30 | +local GEOM = 'geom' -- geometry column (same name in src and dest) |
| 31 | +local ZOOM = 18 -- expire zoom; high so the changed regions are small |
| 32 | +-- Lines merge only when ALL of these columns are equal (NULLs compare equal): |
| 33 | +local GROUP = { 'name', 'ref', 'highway', 'layer' } |
| 34 | +-- Pre-filter: lines not matching are excluded entirely. Use 'true' to merge |
| 35 | +-- all lines. |
| 36 | +local WHERE = 'name IS NOT NULL OR ref IS NOT NULL' |
| 37 | + |
| 38 | +-- --------------------------------------------------------------------------- |
| 39 | +-- Build the SQL fragments from the grouping columns. |
| 40 | +-- --------------------------------------------------------------------------- |
| 41 | +local function join(sep, fn) |
| 42 | + local t = {} |
| 43 | + for _, c in ipairs(GROUP) do t[#t + 1] = fn(c) end |
| 44 | + return table.concat(t, sep) |
| 45 | +end |
| 46 | + |
| 47 | +local V = { |
| 48 | + src = SRC, dest = DEST, expire = EXPIRE, geom = GEOM, zoom = ZOOM, |
| 49 | + where = '(' .. WHERE .. ')', |
| 50 | + group_cols = join(', ', function(c) return '"' .. c .. '"' end), |
| 51 | + group_cols_l = join(', ', function(c) return 'l."' .. c .. '"' end), |
| 52 | + group_cols_gk = join(', ', function(c) return '"' .. c .. '" AS "gk_' .. c .. '"' end), |
| 53 | + group_cols_l_gk = join(', ', function(c) return 'l."' .. c .. '" AS "gk_' .. c .. '"' end), |
| 54 | + group_join = join(' AND ', function(c) return 'l."' .. c .. '" IS NOT DISTINCT FROM n."gk_' .. c .. '"' end), |
| 55 | + group_join_dn = join(' AND ', function(c) return 'd."' .. c .. '" IS NOT DISTINCT FROM n."gk_' .. c .. '"' end), |
| 56 | +} |
| 57 | + |
| 58 | +local function sql(s) |
| 59 | + return (s:gsub('{([%w_]+)}', function(k) |
| 60 | + local v = V[k] |
| 61 | + if v == nil then error("unknown template key '" .. k .. "'") end |
| 62 | + return tostring(v) |
| 63 | + end)) |
| 64 | +end |
| 65 | + |
| 66 | +-- --------------------------------------------------------------------------- |
| 67 | +-- Tables (used by osm2pgsql on import; registered by osm2pgsql-gen) |
| 68 | +-- --------------------------------------------------------------------------- |
| 69 | +local exp_roads = osm2pgsql.define_expire_output({ table = EXPIRE, maxzoom = ZOOM }) |
| 70 | + |
| 71 | +local roads = osm2pgsql.define_table({ |
| 72 | + name = SRC, |
| 73 | + ids = { type = 'way', id_column = 'way_id' }, |
| 74 | + columns = { |
| 75 | + { column = 'name', type = 'text' }, |
| 76 | + { column = 'ref', type = 'text' }, |
| 77 | + { column = 'highway', type = 'text' }, |
| 78 | + { column = 'layer', type = 'int' }, |
| 79 | + -- Any geometry change (add/modify/delete) expires the tiles it covers. |
| 80 | + -- This must be a linestring column: the update walks lines from end |
| 81 | + -- to end with ST_StartPoint()/ST_EndPoint(), which don't do that for |
| 82 | + -- multilinestrings. |
| 83 | + { column = GEOM, type = 'linestring', not_null = true, |
| 84 | + expire = { { output = exp_roads } } }, |
| 85 | + } |
| 86 | +}) |
| 87 | + |
| 88 | +-- The merged lines are maintained by osm2pgsql-gen, so the warning that this |
| 89 | +-- table has no id column is expected. |
| 90 | +osm2pgsql.define_table({ |
| 91 | + name = DEST, |
| 92 | + columns = { |
| 93 | + { column = 'name', type = 'text' }, |
| 94 | + { column = 'ref', type = 'text' }, |
| 95 | + { column = 'highway', type = 'text' }, |
| 96 | + { column = 'layer', type = 'int' }, |
| 97 | + { column = GEOM, type = 'linestring', not_null = true }, |
| 98 | + } |
| 99 | +}) |
| 100 | + |
| 101 | +function osm2pgsql.process_way(object) |
| 102 | + if not object.tags.highway then return end |
| 103 | + roads:insert({ |
| 104 | + name = object.tags.name, |
| 105 | + ref = object.tags.ref, |
| 106 | + highway = object.tags.highway, |
| 107 | + layer = tonumber(object.tags.layer), |
| 108 | + geom = object:as_linestring(), |
| 109 | + }) |
| 110 | +end |
| 111 | + |
| 112 | +-- --------------------------------------------------------------------------- |
| 113 | +-- The generalization, in SQL. process_gen() runs in osm2pgsql-gen; it branches |
| 114 | +-- on osm2pgsql.mode ('create' for the full build, 'append' for updates). |
| 115 | +-- --------------------------------------------------------------------------- |
| 116 | +function osm2pgsql.process_gen() |
| 117 | + if osm2pgsql.mode == 'create' then |
| 118 | + -- One global GROUP BY + ST_LineMerge over the whole source table. |
| 119 | + osm2pgsql.run_sql({ |
| 120 | + description = 'grouped-linemerge: full rebuild', |
| 121 | + transaction = true, |
| 122 | + sql = { |
| 123 | + sql('TRUNCATE {dest}'), |
| 124 | + sql([[INSERT INTO {dest} ({group_cols}, "{geom}") |
| 125 | + SELECT {group_cols}, (ST_Dump(ST_LineMerge(ST_Collect("{geom}")))).geom |
| 126 | + FROM {src} WHERE {where} GROUP BY {group_cols}]]), |
| 127 | + } |
| 128 | + }) |
| 129 | + -- Functional endpoint indexes that make the incremental walk fast. |
| 130 | + osm2pgsql.run_sql({ |
| 131 | + description = 'grouped-linemerge: endpoint indexes', |
| 132 | + sql = { |
| 133 | + sql([[CREATE INDEX IF NOT EXISTS "{src}_glm_startpt" |
| 134 | + ON {src} USING btree (ST_StartPoint("{geom}")) WHERE {where}]]), |
| 135 | + sql([[CREATE INDEX IF NOT EXISTS "{src}_glm_endpt" |
| 136 | + ON {src} USING btree (ST_EndPoint("{geom}")) WHERE {where}]]), |
| 137 | + sql('ANALYZE {dest}'), |
| 138 | + sql('CREATE INDEX IF NOT EXISTS "{dest}_glm_startpt" ON {dest} USING btree (ST_StartPoint("{geom}"))'), |
| 139 | + sql('CREATE INDEX IF NOT EXISTS "{dest}_glm_endpt" ON {dest} USING btree (ST_EndPoint("{geom}"))'), |
| 140 | + } |
| 141 | + }) |
| 142 | + else |
| 143 | + -- Incremental: consume the expired tiles, walk each affected connected |
| 144 | + -- component out from there, and re-merge only those. Everything is one |
| 145 | + -- run_sql with transaction=true so the ON COMMIT DROP temp tables |
| 146 | + -- survive across the steps; if_has_rows makes it a no-op when nothing |
| 147 | + -- expired. |
| 148 | + osm2pgsql.run_sql({ |
| 149 | + description = 'grouped-linemerge: incremental update', |
| 150 | + transaction = true, |
| 151 | + if_has_rows = sql('SELECT 1 FROM {expire} WHERE zoom = {zoom} LIMIT 1'), |
| 152 | + sql = { |
| 153 | + -- 1. expired tiles -> changed-region envelopes |
| 154 | + sql([[CREATE TEMP TABLE _glm_region ON COMMIT DROP AS |
| 155 | + WITH expired AS (DELETE FROM {expire} WHERE zoom = {zoom} RETURNING x, y) |
| 156 | + SELECT ST_TileEnvelope({zoom}, x, y) AS env FROM expired]]), |
| 157 | + 'ANALYZE _glm_region', |
| 158 | + -- 2. walk: seed from lines in the region, flood shared endpoints |
| 159 | + -- within the same grouping key (dedup on (group, point)). |
| 160 | + sql([[CREATE TEMP TABLE _glm_nodes ON COMMIT DROP AS |
| 161 | +WITH RECURSIVE |
| 162 | +seeds AS ( |
| 163 | + SELECT {group_cols_l}, l."{geom}" |
| 164 | + FROM _glm_region r |
| 165 | + JOIN {src} l ON l."{geom}" && r.env AND ST_Intersects(l."{geom}", r.env) |
| 166 | + WHERE {where} |
| 167 | +), |
| 168 | +nodes AS ( |
| 169 | + SELECT b.* FROM ( |
| 170 | + SELECT {group_cols_gk}, ST_StartPoint("{geom}") AS pt FROM seeds |
| 171 | + UNION |
| 172 | + SELECT {group_cols_gk}, ST_EndPoint("{geom}") FROM seeds |
| 173 | + ) b |
| 174 | + UNION |
| 175 | + SELECT {group_cols_l_gk}, |
| 176 | + CASE WHEN ST_StartPoint(l."{geom}") = n.pt |
| 177 | + THEN ST_EndPoint(l."{geom}") ELSE ST_StartPoint(l."{geom}") END |
| 178 | + FROM nodes n |
| 179 | + JOIN {src} l |
| 180 | + ON {group_join} |
| 181 | + AND ( ST_StartPoint(l."{geom}") = n.pt OR ST_EndPoint(l."{geom}") = n.pt ) |
| 182 | + AND {where} |
| 183 | +) |
| 184 | +SELECT * FROM nodes]]), |
| 185 | + 'ANALYZE _glm_nodes', |
| 186 | + -- 3. collect the member lines of those components |
| 187 | + sql([[CREATE TEMP TABLE _glm_ways ON COMMIT DROP AS |
| 188 | +SELECT DISTINCT ON (l.ctid) {group_cols_l}, l."{geom}" |
| 189 | + FROM _glm_nodes n |
| 190 | + JOIN {src} l |
| 191 | + ON {group_join} |
| 192 | + AND ( ST_StartPoint(l."{geom}") = n.pt OR ST_EndPoint(l."{geom}") = n.pt ) |
| 193 | + AND {where} |
| 194 | + ORDER BY l.ctid]]), |
| 195 | + -- 4a. delete stale outputs by reached node (exact endpoint) |
| 196 | + sql([[DELETE FROM {dest} d USING _glm_nodes n |
| 197 | + WHERE {group_join_dn} |
| 198 | + AND ( ST_StartPoint(d."{geom}") = n.pt OR ST_EndPoint(d."{geom}") = n.pt )]]), |
| 199 | + -- 4b. clean up components that vanished entirely (region pass) |
| 200 | + sql([[DELETE FROM {dest} d USING _glm_region r |
| 201 | + WHERE d."{geom}" && r.env AND ST_Intersects(d."{geom}", r.env)]]), |
| 202 | + -- 5. regenerate the affected components |
| 203 | + sql([[INSERT INTO {dest} ({group_cols}, "{geom}") |
| 204 | + SELECT {group_cols}, (ST_Dump(ST_LineMerge(ST_Collect("{geom}")))).geom |
| 205 | + FROM _glm_ways GROUP BY {group_cols}]]), |
| 206 | + } |
| 207 | + }) |
| 208 | + end |
| 209 | +end |
0 commit comments