From dfa50673689e53bdeb48f40fe66006c34e460d32 Mon Sep 17 00:00:00 2001 From: Tobias Date: Wed, 24 Jan 2024 09:30:54 +0100 Subject: [PATCH 1/6] Improve completion message --- app/run-4-process.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/run-4-process.sh b/app/run-4-process.sh index d151ede6..d684a45a 100755 --- a/app/run-4-process.sh +++ b/app/run-4-process.sh @@ -38,6 +38,7 @@ echo "✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ echo -e "\e[1m\e[7m PROCESS – END \e[27m\e[21m – End Time: $(date)\e[0m" echo "Completed:" echo "Development http://localhost:3000/catalog" -echo "Staging https://staging-tiles.radverkehrsatlas.de/" -echo "Production https://tiles.radverkehrsatlas.de" +echo "Staging https://staging-tiles.radverkehrsatlas.de/catalog" +echo "Production https://tiles.radverkehrsatlas.de/catalog" +echo "Test-Map https://data.radverkehrsatlas.de/" echo "✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅ " From 5b4464053c42959ecb2c016b85bdfbb86179dbb9 Mon Sep 17 00:00:00 2001 From: rush42 Date: Mon, 15 Jan 2024 15:59:24 +0100 Subject: [PATCH 2/6] remove all legacy files --- .../legacy_bikelanes/bikelanesPresence.lua | 186 ------------------ app/process/legacy_boundaries/boundaries.lua | 28 --- app/process/legacy_lit/lit.lua | 108 ---------- app/process/legacy_maxspeed/maxspeed.lua | 132 ------------- .../roadClassification.lua | 132 ------------- .../legacy_surfaceQuality/surfaceQuality.lua | 101 ---------- app/run-4-process.sh | 6 - 7 files changed, 693 deletions(-) delete mode 100644 app/process/legacy_bikelanes/bikelanesPresence.lua delete mode 100644 app/process/legacy_boundaries/boundaries.lua delete mode 100644 app/process/legacy_lit/lit.lua delete mode 100644 app/process/legacy_maxspeed/maxspeed.lua delete mode 100644 app/process/legacy_roadClassification/roadClassification.lua delete mode 100644 app/process/legacy_surfaceQuality/surfaceQuality.lua diff --git a/app/process/legacy_bikelanes/bikelanesPresence.lua b/app/process/legacy_bikelanes/bikelanesPresence.lua deleted file mode 100644 index ef2a9535..00000000 --- a/app/process/legacy_bikelanes/bikelanesPresence.lua +++ /dev/null @@ -1,186 +0,0 @@ -package.path = package.path .. ";/app/process/helper/?.lua" -package.path = package.path .. ";/app/process/shared/?.lua" -package.path = package.path .. ";/app/process/roads_bikelanes/bikelanes/?.lua" -require("Set") -require("FilterTags") -require("Metadata") -require("HighwayClasses") -require("RoadWidth") -require("ExcludeHighways") -require("IsFresh") -require("categories") -require("transformations") -require("JoinSets") -require("PrintTable") --- require("IntoExcludeTable") -require("ConvertCyclewayOppositeSchema") - - -local presenceTable = osm2pgsql.define_table({ - name = 'bikelanesPresence', - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'linestring' }, - { column = 'left', type = 'text' }, - { column = 'self', type = 'text' }, - { column = 'right', type = 'text' }, - { column = 'meta', type = 'jsonb' } - } -}) - --- local excludeTable = osm2pgsql.define_table({ --- name = 'bikelanes_excluded', --- ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, --- columns = { --- { column = 'tags', type = 'jsonb' }, --- { column = 'meta', type = 'jsonb' }, --- { column = 'reason', type = 'text' }, --- { column = 'geom', type = 'linestring' }, --- } --- }) - --- whitelist of tags we want to insert intro the DB -local allowed_tags = Set({ - '_parent_highway', - 'access', - 'bicycle_road', - 'bicycle', - 'conditional', - 'cycleway', - 'cycleway:lane', -- 'advisory', 'exclusive' - 'dual_carriageway', - 'foot', - 'footway', - 'highway', - 'is_sidepath', - 'name', - 'oneway', -- we use oneway:bicycle=no (which is transformed to oneway=no) to add a notice in the UI about two way cycleways in one geometry - 'prefix', - 'segregated', - 'side', - 'smoothness', - 'surface:colour', - 'surface', - 'traffic_sign', - 'width', - 'bicycle:lanes', - 'cycleway:lanes', - 'separation', - 'separation:left', - 'separation:right', - 'lane', -- 'cycleway:SIDE:lane' -}) - -local SIDES = { LEFT_SIGN, CENTER_SIGN, RIGHT_SIGN } - -function osm2pgsql.process_way(object) - -- filter highway classes - local allowed_highways = JoinSets({ HighwayClasses, MajorRoadClasses, MinorRoadClasses, PathClasses }) - if not object.tags.highway or not allowed_highways[object.tags.highway] then return end - - local exclude, _ = ExcludeHighways(object.tags) - if exclude then - -- IntoExcludeTable(excludeTable, object, reason) - return - end - - local tags = object.tags - local meta = Metadata(object) - - ConvertCyclewayOppositeSchema(tags) - - -- Our atlas-app inspector should be explicit about tagging that OSM considers default/implicit - if tags.bicycle_road == 'yes' then - tags.oneway = tags.oneway or 'implicit_no' - end - - -- transformations - local footwayTransformation = { - highway = "footway", - prefix = "sidewalk", - filter = function(tags) - return not (tags.footway == 'no' or tags.footway == 'separate') - end - } - -- cycleway transformer: - local cyclewayTransformation = { - highway = "cycleway", - prefix = "cycleway", - } - local transformations = { cyclewayTransformation, footwayTransformation } -- order matters for presence - - -- generate cycleways from center line tagging, also includes the original object with `sign = 0` - local cycleways = GetTransformedObjects(tags, transformations); - local presence = {} -- table holding the presence (per way object) - local width = RoadWidth(tags) - for _, cycleway in pairs(cycleways) do - local sign = cycleway.sign - local onlyPresent = OnlyPresent(cycleway) -- these are categories defining the presence of data - if onlyPresent ~= nil then - presence[sign] = presence[sign] or onlyPresent - else - local category = CategorizeBikelane(cycleway) - if category ~= nil then - FilterTags(cycleway, allowed_tags) - - local freshTag = "check_date" - if cycleway.prefix then - freshTag = "check_date:" .. cycleway.prefix - end - - -- Our atlas-app inspector should be explicit about tagging that OSM considers default/implicit - cycleway.oneway = cycleway.oneway or 'implicit_yes' - - presence[sign] = presence[sign] or category - end - end - end - - -- Below is presence logic - -- Filter ways where we dont expect bicycle infrastructure - -- TODO: filter on surface and traffic zone and maxspeed (maybe wait for maxspeed PR) - if (MinorRoadClasses[tags.highway] and tags.highway ~= 'service') or presence[CENTER_SIGN] then - -- set the nil values to 'not_expected', for all minor roads and complete data - for _, side in pairs(SIDES) do presence[side] = presence[side] or NOT_EXPECTED end - elseif not (presence[CENTER_SIGN] or presence[RIGHT_SIGN] or presence[LEFT_SIGN]) then - if not MajorRoadClasses[tags.highway] then - -- IntoExcludeTable(excludeTable, object, "no infrastructure expected for highway type: " .. tags.highway) - return - elseif tags.motorroad or tags.expressway then - -- IntoExcludeTable(excludeTable, object, "no infrastructure expected for motorroad and express way") - return - -- elseif tags.maxspeed and tags.maxspeed <= 20 then - -- intoExcludeTable(object, "no infrastructure expected for max speed <= 20 kmh") - -- return - end - elseif (presence[RIGHT_SIGN] or presence[LEFT_SIGN]) then - presence[CENTER_SIGN] = presence[CENTER_SIGN] or NOT_EXPECTED - end - if tags.oneway == 'yes' and tags['oneway:bicycle'] ~= 'no' then - presence[LEFT_SIGN] = presence[LEFT_SIGN] or NOT_EXPECTED - end - - -- replace all nil values with 'missing' - for _, side in pairs(SIDES) do presence[side] = presence[side] or "missing" end - - local allowed_tags_presence = Set({ - 'left', - 'right', - 'self', - 'name', - 'highway', - 'oneway', - 'dual_carriageway', - }) - FilterTags(tags, allowed_tags_presence) - - presenceTable:insert({ - tags = tags, - geom = object:as_linestring(), - left = presence[LEFT_SIGN], - self = presence[CENTER_SIGN], - right = presence[RIGHT_SIGN], - meta = meta, - }) -end diff --git a/app/process/legacy_boundaries/boundaries.lua b/app/process/legacy_boundaries/boundaries.lua deleted file mode 100644 index a04129d6..00000000 --- a/app/process/legacy_boundaries/boundaries.lua +++ /dev/null @@ -1,28 +0,0 @@ -local srid = 4326 - --- TODO: This is the only table that does not use jsonb to store the tags. --- Which means it breaks when we try to use our export script. --- It also lacks a jsonb meta column. --- See also db_configuration.py. - -local table = osm2pgsql.define_table({ - name = 'legacy_boundaries', - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'name', type = 'text' }, - { column = 'admin_level', sql_type = 'numeric' }, - { column = 'geom', type = 'geometry', projection = srid }, - { column = 'area', type = 'area' }, - } -}) - -function osm2pgsql.process_relation(object) - if object.tags.type == 'boundary' and - object.tags.boundary == "administrative" then - table:add_row { - name = object.tags["name"], - admin_level = object.tags["admin_level"], - geom = { create = "area" } - } - end -end diff --git a/app/process/legacy_lit/lit.lua b/app/process/legacy_lit/lit.lua deleted file mode 100644 index 832d4285..00000000 --- a/app/process/legacy_lit/lit.lua +++ /dev/null @@ -1,108 +0,0 @@ -package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua" -require("Set") -require("FilterTags") -require("MergeArray") -require("JoinSets") -require("Metadata") -require("HighwayClasses") -require("ExcludeHighways") -require("ExcludeByWidth") -require("IsFresh") - -local table = osm2pgsql.define_table({ - name = 'lit', - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'category', type = 'text' }, - { column = 'tags', type = 'jsonb' }, - { column = 'meta', type = 'jsonb' }, - { column = 'geom', type = 'linestring' }, - } -}) - --- Notes --- ===== --- Which highways do we look at? --- We should include sidewalks. --- But ideally, we would exclude ways that are not network-relevant like cemetary footways - --- We ignore street lamps: --- https://wiki.openstreetmap.org/wiki/Tag:highway%3Dstreet_lamp is not relevant for this data set - --- About nodes and relations: --- process_node: We only look at highways --- process_relation: We might need to add relations later; but for now ways should be enought - --- TODO: Process sidewalk:*, cycleway:* data othe centerline --- Right now, we only look at the primary data. --- We need to improve this, to copy the ways for sidewalks and cycleways that are not yet mapped separately. And apply the data on those ways as well. --- Could we do this, by adding a _processing_instructions="move left". --- We add those ways twice to the data … and post-process the in SQL? - -function osm2pgsql.process_way(object) - if not object.tags.highway then return end - - local allowed_highways = JoinSets({ HighwayClasses, MajorRoadClasses, MinorRoadClasses, PathClasses }) - -- values that we would allow, but skip here: - -- "construction", "planned", "proposed", "platform" (Haltestellen) - if not allowed_highways[object.tags.highway] then return end - if ExcludeHighways(object.tags) or ExcludeByWidth(object.tags, 2.1) then - return - end - - -- https://wiki.openstreetmap.org/wiki/Key:lit - - -- Categorize the data in three groups: "lit", "unlit", "special" - local category = nil - if object.tags.lit == nil then - object.tags.is_present = false - else - object.tags.is_present = true - category = "special" - if (object.tags.lit == "yes") then - category = "lit" - end - if (object.tags.lit == "no") then - category = "unlit" - end - end - - -- Normalize name info for sidepath' - -- TODO: Extact into helper - object.tags.name = object.tags.name or object.tags['is_sidepath:of:name'] - - local allowed_tags = Set({ - "access", - "area", - "category", - "check_date:lit", - "footway", - "highway", - "is_present", - "is_sidepath", - "lit", - "surface", - "smoothness", - "name", - "service", - "width", -- experimental - "sidewalk:width", -- experimental - "cycleway:width", -- experimental - }) - FilterTags(object.tags, allowed_tags) - - -- Freshness of data (AFTER `FilterTags`!) - if (object.tags.is_present == true) then - IsFresh(object, 'check_date:lit', object.tags) - end - - if object.tags._exclude then - else - table:insert({ - category = category, - tags = object.tags, - meta = Metadata(object), - geom = object:as_linestring() - }) - end -end diff --git a/app/process/legacy_maxspeed/maxspeed.lua b/app/process/legacy_maxspeed/maxspeed.lua deleted file mode 100644 index 35abd02a..00000000 --- a/app/process/legacy_maxspeed/maxspeed.lua +++ /dev/null @@ -1,132 +0,0 @@ -package.path = package.path .. ";/app/process/helper/?.lua" -package.path = package.path .. ";/app/process/shared/?.lua" -package.path = package.path .. ";/app/process/roads_bikelanes/maxspeed/?.lua" -require("ExcludeHighways") -require("FilterTags") -require("HighwayClasses") -require("IntoExcludeTable") -require("IsFresh") -require("JoinSets") -require("MaxspeedDirect") -require("MaxspeedFromZone") -require("Metadata") -require("Set") - -local table = osm2pgsql.define_table({ - name = "maxspeed", - ids = { type = "any", id_column = "osm_id", type_column = "osm_type" }, - columns = { - { column = "tags", type = "jsonb" }, - { column = "meta", type = "jsonb" }, - { column = "geom", type = "linestring" } - } -}) - --- Roads that we exlude from our analysis -local excludeTable = osm2pgsql.define_table({ - name = "maxspeed_excluded", - ids = { type = "any", id_column = "osm_id", type_column = "osm_type" }, - columns = { - { column = "tags", type = "jsonb" }, - { column = "meta", type = "jsonb" }, - { column = "reason", type = "text" }, - { column = "geom", type = "linestring" } - } -}) - --- Define tables with all bicycle related roads that currently dont have speed values -local missingTable = osm2pgsql.define_table({ - name = "_maxspeed_missing", - ids = { type = "any", id_column = "osm_id", type_column = "osm_type" }, - columns = { - { column = "tags", type = "jsonb" }, - { column = "meta", type = "jsonb" }, - { column = "geom", type = "linestring" } - } -}) - -function osm2pgsql.process_way(object) - local allowed_highways = JoinSets({ MajorRoadClasses, MinorRoadClasses }) - if not allowed_highways[object.tags.highway] then - return - end - - local tags = object.tags - tags.raw_maxspeed = tags.maxspeed -- Preserve original value since we use `maxspeed` for our processed data - - local exclude, reason = ExcludeHighways(tags) - if exclude then - IntoExcludeTable(excludeTable, object, reason) - return - end - - -- TODO: Why would we want to exclude this based on this tag? (Tobias) - -- if tags.bicycle == "no" then - -- IntoExcludeTable(table, object, "no bikes allowed") - -- return - -- end - - -- Try to find maxspeed information in the following order: - -- 1. `maxspeed` tag - -- 2. maxspeed zones tags - -- 3. highway type - -- 4. SQL: intersecting landuse - local maxspeed, source, confidence = MaxspeedDirect(tags) - - if maxspeed == nil then - maxspeed, source, confidence = MaxspeedFromZone(tags) - end - - if maxspeed == nil then - local highway_speeds = { - ["living_street"] = 7 - } - if highway_speeds[tags.highway] then - maxspeed = highway_speeds[tags.highway] - source = "inferred_from_highway" - confidence = 'high' -- living_street is 'high', others would be 'medium - end - end - - -- all tags that are shown on the application - local allowed_tags = { - "_todo", - "name", - "highway", - "bicycle_road", - "raw_maxspeed", - "maxspeed:backward", - "maxspeed:forward", - "maxspeed:conditional", - "maxspeed:type", - "zone:maxspeed", - "source:maxspeed", - "traffic_sign", - "checkdate:maxspeed", - } - FilterTags(tags, Set(allowed_tags)) - - -- Freshness of data (AFTER `FilterTags`!) - IsFresh(object, "check_date:maxspeed", tags) - - tags.maxspeed = maxspeed - tags.maxspeed_source = source - tags.maxspeed_confidence = confidence - - local meta = Metadata(object) - - if maxspeed ~= nil then - table:insert({ - tags = tags, - meta = meta, - geom = object:as_linestring(), - }) - return - end - - missingTable:insert({ - tags = object.tags, - meta = meta, - geom = object:as_linestring(), - }) -end diff --git a/app/process/legacy_roadClassification/roadClassification.lua b/app/process/legacy_roadClassification/roadClassification.lua deleted file mode 100644 index 6e57dbb7..00000000 --- a/app/process/legacy_roadClassification/roadClassification.lua +++ /dev/null @@ -1,132 +0,0 @@ -package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua" -require("Set") -require("FilterTags") -require("MergeArray") -require("Metadata") -require("HighwayClasses") -require("JoinSets") -require("ExcludeHighways") -require("ExcludeByWidth") -require("IntoExcludeTable") -require("ConvertCyclewayOppositeSchema") - -local table = osm2pgsql.define_table({ - name = 'roadClassification', - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'tags', type = 'jsonb' }, - { column = 'meta', type = 'jsonb' }, - { column = 'geom', type = 'linestring' }, - } -}) - -local excludeTable = osm2pgsql.define_table({ - name = 'roadClassification_excluded', - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'tags', type = 'jsonb' }, - { column = 'meta', type = 'jsonb' }, - { column = 'reason', type = 'text' }, - { column = 'geom', type = 'linestring' }, - } -}) - -function osm2pgsql.process_way(object) - if not object.tags.highway then return end - - local allowed_highways = JoinSets({ HighwayClasses, MajorRoadClasses, MinorRoadClasses, PathClasses }) - -- Values that we would allow, but skip here: - -- "construction", "planned", "proposed", "platform" (Haltestellen), - -- "rest_area" (https://wiki.openstreetmap.org/wiki/DE:Tag:highway=rest%20area) - if not allowed_highways[object.tags.highway] then return end - - local exclude, reason = ExcludeHighways(object.tags) - if exclude then - IntoExcludeTable(excludeTable, object, reason) - return - end - - exclude, reason = ExcludeByWidth(object.tags, 2.1) - if exclude then - IntoExcludeTable(excludeTable, object, reason) - return - end - - if object.tags.area == 'yes' then - IntoExcludeTable(excludeTable, object, "Exclude `area=yes`") - return - end - - -- Exclude sidewalk `(highway=footway) + footway=sidewalk` - -- Including "Fahrrad frei" https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:1022-10 - if object.tags.footway == "sidewalk" then - IntoExcludeTable(excludeTable, object, "Exclude `footway=sidewalk`") - return - end - - -- Exclude `is_sidepath=yes` - -- Including "Fahrrad frei" https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:1022-10 - if object.tags.is_sidepath == "yes" then - IntoExcludeTable(excludeTable, object, "Exclude `is_sidepath=yes`") - return - end - - ConvertCyclewayOppositeSchema(object.tags) - - -- https://wiki.openstreetmap.org/wiki/DE:Key:highway - -- We use the OSM value as category, but have a few special cases below. - object.tags.category = object.tags.highway - if object.tags.highway == "road" then - object.tags.category = "unspecified_road_category" - end - - -- https://wiki.openstreetmap.org/wiki/DE:Key:service - if object.tags.highway == "service" then - -- Fallback: - object.tags.category = 'service_uncategorized' - -- https://taginfo.openstreetmap.org/keys/service#values - if object.tags.service == nil then - object.tags.category = "service_road" - end - if object.tags.service == 'alley' then - object.tags.category = "service_alley" - end - if object.tags.service == 'driveway' then - object.tags.category = "service_driveway" - end - if object.tags.service == 'parking_aisle' then - object.tags.category = "service_parking_aisle" - end - end - - if object.tags.bicycle_road == 'yes' then - -- traffic_sign=DE:244.1 - object.tags.category = "bicycle_road" - end - - if object.tags.highway == 'steps' then - -- Simplify steps as part of the walkable network - object.tags.category = "footway" - end - - if object.tags.oneway == 'yes' then - if object.tags['oneway:bicycle'] == 'no' then - object.tags.oneway = 'car_not_bike' - else - object.tags.oneway = 'car_and_bike' - end - if object.tags.dual_carriageway == "yes" then - object.tags.oneway = object.tags.oneway .. '_dual_carriageway' - end - end - - local allowed_tags = Set({ "category", "name", "highway", "footway", "access", "service", - "is_sidepath", "maxspeed", "surface", "smoothness", "oneway" }) - FilterTags(object.tags, allowed_tags) - - table:insert({ - tags = object.tags, - meta = Metadata(object), - geom = object:as_linestring() - }) -end diff --git a/app/process/legacy_surfaceQuality/surfaceQuality.lua b/app/process/legacy_surfaceQuality/surfaceQuality.lua deleted file mode 100644 index 12f1c01c..00000000 --- a/app/process/legacy_surfaceQuality/surfaceQuality.lua +++ /dev/null @@ -1,101 +0,0 @@ -package.path = package.path .. ";/app/process/helper/?.lua" -package.path = package.path .. ";/app/process/shared/?.lua" -package.path = package.path .. ";/app/process/roads_bikelanes/surfaceQuality/?.lua" -require("ExcludeHighways") -require("ExcludeByWidth") -require("FilterTags") -require("HighwayClasses") -require("IntoExcludeTable") -require("IsFresh") -require("JoinSets") -require("DeriveSurface") -require("DeriveSmoothness") -require("Metadata") -require("Set") - -local table = osm2pgsql.define_table({ - name = "surfaceQuality", - ids = { type = "any", id_column = "osm_id", type_column = "osm_type" }, - columns = { - { column = "tags", type = "jsonb" }, - { column = "meta", type = "jsonb" }, - { column = "geom", type = "linestring" } - } -}) - --- Roads that we exlude from our analysis -local excludeTable = osm2pgsql.define_table({ - name = "surfaceQuality_excluded", - ids = { type = "any", id_column = "osm_id", type_column = "osm_type" }, - columns = { - { column = "tags", type = "jsonb" }, - { column = "meta", type = "jsonb" }, - { column = "reason", type = "text" }, - { column = "geom", type = "linestring" } - } -}) - -function osm2pgsql.process_way(object) - -- Same as roadClassification, except for `HighwayClasses` - local allowed_highways = JoinSets({ MajorRoadClasses, MinorRoadClasses, PathClasses }) - if not allowed_highways[object.tags.highway] then return end - - local tags = object.tags - -- tags.raw_surface = tags.surface -- Preserve original value since we use `surface` for our processed data - -- tags.raw_smoothness = tags.smoothness -- Preserve original value since we use `smoothness` for our processed data - - local exclude, reason = ExcludeHighways(object.tags) - if exclude then - IntoExcludeTable(excludeTable, object, reason) - return - end - - exclude, reason = ExcludeByWidth(object.tags, 2.1) - if exclude then - IntoExcludeTable(excludeTable, object, reason) - return - end - - if object.tags.area == 'yes' then - IntoExcludeTable(excludeTable, object, "Exclude `area=yes`") - return - end - - local surface, surface_source = DeriveSurface(tags) - - local todo = nil - local smoothness, smoothness_source, smoothness_confidence = DeriveSmoothness(tags) - - -- all tags that are shown on the application - local allowed_tags = { - "name", - "highway", - "raw_surface", - "raw_smoothness", - "checkdate:surface", - "checkdate:smoothness", - } - FilterTags(tags, Set(allowed_tags)) - - -- Freshness of data (AFTER `FilterTags`!) - IsFresh(object, "check_date:surface", tags, "surface") - IsFresh(object, "check_date:smoothness", tags, "smoothness") - - tags.surface = surface - tags.surface_source = surface_source - -- tags.surface_confidence = surface_confidence -- only needed after we extended the surface normalization - - tags.smoothness = smoothness - tags.smoothness_source = smoothness_source - tags.smoothness_confidence = smoothness_confidence - tags._todo = todo - - tags.is_present_surface = tags.surface ~= nil - tags.is_present_smoothness = tags.smoothness ~= nil - - table:insert({ - tags = tags, - meta = Metadata(object), - geom = object:as_linestring(), - }) -end diff --git a/app/run-4-process.sh b/app/run-4-process.sh index d684a45a..b2f641d5 100755 --- a/app/run-4-process.sh +++ b/app/run-4-process.sh @@ -14,12 +14,6 @@ run_psql "roads_bikelanes/bikelanes/bikelanes" run_lua "bicycleParking/bicycleParking" -# run_lua "legacy_bikelanes/bikelanesPresence" -# run_lua "legacy_surfaceQuality/surfaceQuality" -# run_lua "legacy_roadClassification/roadClassification" -# run_lua "legacy_maxspeed/maxspeed" -# run_lua "legacy_lit/lit" - run_lua "trafficSigns/trafficSigns" run_psql "trafficSigns/trafficSigns" From 1d195bf954de92e28757c270b7d6620ae0b86600 Mon Sep 17 00:00:00 2001 From: rush42 Date: Mon, 15 Jan 2024 16:08:00 +0100 Subject: [PATCH 3/6] remove API docker --- .env.example | 2 - .github/workflows/deployment.production.yml | 4 +- .github/workflows/deployment.staging.yml | 4 +- .github/workflows/deployment.yml | 28 +-- api.Dockerfile | 7 - api/INIT_FUNCTIONS.sql | 24 --- api/INIT_VERIFICATION_VIEWS.sql | 65 ------- api/README.md | 3 - api/db.py | 11 -- api/db_configuration.py | 45 ----- api/generate_test_data.py | 40 ---- api/main.py | 185 ------------------- api/requirements.txt | 3 - app/{run-7-analysis.sh => run-6-analysis.sh} | 0 app/run-6-api.sh | 9 - app/{run-8-metadata.sh => run-7-metadata.sh} | 0 app/run.sh | 14 +- docker-compose.development.yml | 16 -- docker-compose.yml | 25 +-- 19 files changed, 10 insertions(+), 475 deletions(-) delete mode 100644 api.Dockerfile delete mode 100644 api/INIT_FUNCTIONS.sql delete mode 100644 api/INIT_VERIFICATION_VIEWS.sql delete mode 100644 api/README.md delete mode 100644 api/db.py delete mode 100644 api/db_configuration.py delete mode 100644 api/generate_test_data.py delete mode 100644 api/main.py delete mode 100644 api/requirements.txt rename app/{run-7-analysis.sh => run-6-analysis.sh} (100%) delete mode 100755 app/run-6-api.sh rename app/{run-8-metadata.sh => run-7-metadata.sh} (100%) diff --git a/.env.example b/.env.example index 8f08e01a..16c03250 100644 --- a/.env.example +++ b/.env.example @@ -15,8 +15,6 @@ SKIP_TAG_FILTER=0 # skips tag filtering DEBUG=1 # enables the generation of exclude tables ID_FILTER='' # if not empty only the objects with these ids are processed. See https://docs.osmcode.org/osmium/latest/osmium-getid.html -# Internal API secret -API_SECRET=myapisecret # Token for Synology log. Leave blank to disable logging SYNOLOGY_LOG_TOKEN= diff --git a/.github/workflows/deployment.production.yml b/.github/workflows/deployment.production.yml index 19b5142f..d4fc00d4 100644 --- a/.github/workflows/deployment.production.yml +++ b/.github/workflows/deployment.production.yml @@ -10,9 +10,7 @@ jobs: uses: ./.github/workflows/deployment.yml with: ENVIRONMENT: production - URL: https://tiles.radverkehrsatlas.de - TILES_URL: tiles.radverkehrsatlas.de - API_URL: api.radverkehrsatlas.de + URL: tiles.radverkehrsatlas.de secrets: SERVICE_NAME: ${{ secrets.SERVICE_NAME }} DATABASE_NAME: ${{ secrets.DATABASE_NAME }} diff --git a/.github/workflows/deployment.staging.yml b/.github/workflows/deployment.staging.yml index c080a1bf..17dffafa 100644 --- a/.github/workflows/deployment.staging.yml +++ b/.github/workflows/deployment.staging.yml @@ -10,9 +10,7 @@ jobs: uses: ./.github/workflows/deployment.yml with: ENVIRONMENT: staging - URL: https://staging-tiles.radverkehrsatlas.de - TILES_URL: staging-tiles.radverkehrsatlas.de - API_URL: staging-api.radverkehrsatlas.de + URL: staging-tiles.radverkehrsatlas.de secrets: SERVICE_NAME: ${{ secrets.SERVICE_NAME }} DATABASE_NAME: ${{ secrets.DATABASE_NAME }} diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 078cd237..bef2fca0 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -36,19 +36,13 @@ on: URL: type: string required: true - TILES_URL: - type: string - required: true - API_URL: - type: string - required: true jobs: build_and_deploy: runs-on: ubuntu-latest environment: name: ${{ inputs.ENVIRONMENT }} - url: ${{ inputs.URL }} + url: https://${{ inputs.URL }} steps: - uses: actions/checkout@v4 @@ -89,22 +83,6 @@ jobs: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/n0p8j4k5/ docker push public.ecr.aws/n0p8j4k5/atlas/app:${{ github.sha }} - - name: Build api image - uses: docker/build-push-action@v5 - with: - context: . - file: ./api.Dockerfile - push: false - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - tags: public.ecr.aws/n0p8j4k5/atlas/api:${{ github.sha }} - - - name: Push API image - run: | - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/n0p8j4k5/ - docker push public.ecr.aws/n0p8j4k5/atlas/api:${{ github.sha }} - - name: Copy files to server uses: appleboy/scp-action@master with: @@ -134,10 +112,8 @@ jobs: echo PGPASSWORD='${{ secrets.DATABASE_PASSWORD }}' >> .env echo PGDATABASE='${{ secrets.DATABASE_NAME }}' >> .env echo OSM_DOWNLOAD_URL='${{ vars.OSM_DOWNLOAD_URL }}' >> .env - echo TILES_URL='${{ inputs.TILES_URL }}' >> .env - echo API_URL='${{ inputs.API_URL }}' >> .env + echo URL='${{ inputs.URL }}' >> .env echo GITHUB_SHA='${{ github.sha }}' >> .env - echo API_SECRET=$(echo $RANDOM | md5sum | head -c 20) >> .env echo "Reload containers" docker compose -f docker-compose.traefik.yml up -d docker compose up -d diff --git a/api.Dockerfile b/api.Dockerfile deleted file mode 100644 index 3c78216a..00000000 --- a/api.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ghcr.io/multi-py/python-uvicorn:py3.11-0.24.0 - -COPY ./api/requirements.txt /app/requirements.txt - -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt - -COPY ./api /app diff --git a/api/INIT_FUNCTIONS.sql b/api/INIT_FUNCTIONS.sql deleted file mode 100644 index d92d87a1..00000000 --- a/api/INIT_FUNCTIONS.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE OR REPLACE FUNCTION public. {function_name}(region geometry) - RETURNS json - LANGUAGE sql - AS $function$ - SELECT - json_build_object('type', 'FeatureCollection', 'license', 'ODbL 1.0, https://opendatacommons.org/licenses/odbl/', - 'attribution', 'OpenStreetMap, https://www.openstreetmap.org/copyright; Radverkehrsatlas.de', 'features', json_agg(features.feature)) - FROM( - SELECT - jsonb_build_object('type', 'Feature', 'geometry', - ST_AsGeoJSON(ST_Transform(geom, 4326))::jsonb, - -- Reminder: All tables that can be exported are required to have a those columns - 'properties', jsonb_build_object('osm_id', inputs.osm_id) || - jsonb_build_object('osm_type', inputs.osm_type) || inputs.meta || - inputs.tags) AS feature - FROM( - SELECT - * - FROM - "{table_name}" - WHERE - ST_Transform(geom, 4326) && region) inputs) features; - -$function$; diff --git a/api/INIT_VERIFICATION_VIEWS.sql b/api/INIT_VERIFICATION_VIEWS.sql deleted file mode 100644 index 68e92e6f..00000000 --- a/api/INIT_VERIFICATION_VIEWS.sql +++ /dev/null @@ -1,65 +0,0 @@ -DROP INDEX IF EXISTS {geometry_table}_osm_id_idx; - -CREATE INDEX {geometry_table}_osm_id_idx ON {geometry_table} USING btree(osm_id); - --- TODO Cleanup: This is now done by atlas-app, Prisma --- CREATE TABLE IF NOT EXISTS public.{verification_table} ( --- osm_type varchar NOT NULL, --- -- osm_type bpchar(1) NOT NULL, --- osm_id bigint NOT NULL, --- -- osm_id int8 NOT NULL, --- verified_at timestamp NOT NULL, --- verified_by bigint, --- verified varchar NULL, --- comment text --- ); --- TODO: ALTER TABLE {verification_table} ALTER COLUMN osm_type TYPE bpchar(1) USING osm_type::bpchar; -DROP TABLE IF EXISTS public. "{joined_table}"; - -CREATE TABLE public. {joined_table} AS ( - SELECT - g.osm_type, - g.osm_id, - g.tags, - g.meta, - g.geom, - v.verified_at, - v.verified_by, - v.verified - FROM - public. "{geometry_table}" g - LEFT JOIN ( SELECT DISTINCT ON (v.osm_id) - * - FROM - prisma. "{verification_table}" v - ORDER BY - v.osm_id, verified_at DESC) v ON g.osm_id = v.osm_id -); - -CREATE INDEX {joined_table}_geom_idx ON {joined_table} USING gist(geom) WITH (fillfactor = '100'); - -CREATE INDEX {joined_table}_osm_id_idx ON {joined_table} USING btree(osm_id); - -CREATE OR REPLACE FUNCTION "atlas_update_{joined_table}"() - RETURNS TRIGGER - LANGUAGE PLPGSQL - AS $$ -BEGIN - UPDATE - public. "{joined_table}" - SET - verified_at = NEW.verified_at, - verified_by = NEW.verified_by, - verified = NEW.verified - WHERE - osm_id = NEW.osm_id; - RETURN NEW; -END; -$$; - -DROP TRIGGER IF EXISTS "atlas_update_{joined_table}" ON prisma. "{verification_table}"; - -CREATE TRIGGER "atlas_update_{joined_table}" - AFTER INSERT ON prisma. "{verification_table}" - FOR EACH ROW - EXECUTE PROCEDURE "atlas_update_{joined_table}"(); diff --git a/api/README.md b/api/README.md deleted file mode 100644 index b2505dee..00000000 --- a/api/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -Run it `uvicorn api:app --reload`. diff --git a/api/db.py b/api/db.py deleted file mode 100644 index b5482396..00000000 --- a/api/db.py +++ /dev/null @@ -1,11 +0,0 @@ -import os - - -database_server = os.environ.get('PGHOST', "") -database_name = os.environ.get('PGDATABASE', "postgres") -database_user = os.environ.get('PGUSER', "postgres") -database_pw = os.environ.get('PGPASSWORD', "") -database_port = os.environ.get('PGPORT', "5432") -api_secret = os.environ.get('API_SECRET', "") - -conn_string = "host=%s dbname=%s user=%s password=%s port=%s" %(database_server, database_name, database_user, database_pw, database_port) diff --git a/api/db_configuration.py b/api/db_configuration.py deleted file mode 100644 index 86065fc0..00000000 --- a/api/db_configuration.py +++ /dev/null @@ -1,45 +0,0 @@ -from enum import Enum - -# Helper to retrieve the name of the verification table -# The verification table is owned by prisma and part of the prisma schema. -# See https://github.com/FixMyBerlin/atlas-app/blob/develop/db/schema.prisma -def verification_table(table_name: str): - # needs to be wrapped in quotes to preserve capitalization - return f'"{table_name}Verification"' - -def verified_table(table_name: str): - return f'{table_name}_verified' - -def export_function(table_name: str): - return f'atlas_export_geojson_{table_name.lower()}' - -# Te list of DB Tables which support exports. -ExportTable = Enum('Export Table', [(export_function(name), name) for name in [ - "barrierAreas", - "barrierLines", - "bikelanes_verified", - "bikelanes", - "boundaries", - "roads", - "landuse", - "places", - "poiClassification", - "publicTransport", - "trafficSigns", - "bicycleParking_points", - "bicycleParking_areas" -]]) - -# The list of DB Tables that support verification. -# For each table `a` in the list we create a verification table `a_verification` and a view which joins both -# TODO: This part is not used ATM. Instead in api/main.py we pass the names manually. -# The issue is, that the Verification Table is now owned by Prisma with consols the naming schema. -# Also we have it as BikelaneVerificaition (singular) in Prisma. -# We need to either fix our conventions or change this Enum to an object that holds the name for all -# three tables `verification_table`, `geometry_table`, `joined_table` -VerificationTable = Enum('Verication Table', [(verification_table(name), name) for name in ["Bikelane"]]) - -# main.py: Used as an allow list to guard the /verify/* API -# `atlas-app` only uses approved, rejected for now. -VerifiedState = Enum('Verify State', [(name, name) for name in ['approved', 'rejected', 'undefined'] -]) diff --git a/api/generate_test_data.py b/api/generate_test_data.py deleted file mode 100644 index 67028257..00000000 --- a/api/generate_test_data.py +++ /dev/null @@ -1,40 +0,0 @@ -# this is _supposed_ to be quick'n'dirty -import psycopg -from datetime import datetime, timedelta -from random import randint -from db import conn_string - -# TODO: 'lit' does not have a verification table anymore -geometry_table = 'lit' -verification_table = 'lit_verification' - -num_chunks = 100 -chunk_size = 10000 - -conn = psycopg.connect(conn_string) -cursor = conn.cursor() -cursor.execute(f'select distinct osm_id from {geometry_table};') -ids = cursor.fetchall() -ids = [id[0] for id in ids] - -cursor.execute(f'TRUNCATE TABLE prisma."{verification_table}";') - -start_dt = datetime(year=2022, month=1, day=1) -ms = 11 * 30 * 24 * 60 * 60 * 1000 # 11 months - -print(f'inserting {num_chunks * chunk_size} rows...') - -for chunk in range(num_chunks): - print(f'chunk {chunk} / {num_chunks}') - sql = f'INSERT INTO prisma."{verification_table}" (osm_type,osm_id,verified_at,verified_by,verified) VALUES\n' - for line in range(chunk_size): - id = ids[randint(0, len(ids) - 1)] - dt = (start_dt + timedelta(milliseconds=randint(0, ms))).isoformat() - status = 'approved' if randint(0, 1) else 'rejected' - line = f" ('W',{id},'{dt}',NULL,'{status}'),\n" - sql += line - sql = sql[:-2] + ';\n' - sql += 'end;\n' - cursor.execute(sql) - -print('done.') diff --git a/api/main.py b/api/main.py deleted file mode 100644 index 6221443c..00000000 --- a/api/main.py +++ /dev/null @@ -1,185 +0,0 @@ -from fastapi import FastAPI, Header, Response, Request, HTTPException, Query -from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.responses import FileResponse -from typing import Union, Annotated -from psycopg import sql -from psycopg.rows import dict_row -from db_configuration import VerificationTable, ExportTable, VerifiedState, verified_table -from db import conn_string, api_secret -import psycopg -from pathlib import Path - - -app = FastAPI( - title="RadverkehrsatlasTools", - description="Use and interact with OSM data from FixMyCity", - version="0.1.0", - license_info={ - "name":"ODbL 1.0", - "url": "https://opendatacommons.org/licenses/odbl/" - } -) - -origins = [ - "http://localhost:5173", - "http://127.0.0.1:5173", # atlas-app with `npm run dev` - "http://127.0.0.1:3000", # atlas-app with `npm run start` - "https://develop--radverkehrsatlas.netlify.app", # Legacy - "https://staging.radverkehrsatlas.de", - "https://radverkehrsatlas.de", - "https://www.radverkehrsatlas.de", -] - -app.add_middleware( - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -app.add_middleware(GZipMiddleware, minimum_size=1000) - -@app.get("/export/{type_name}") -async def export_bbox(response: Response, type_name: ExportTable, minlon: float= 13.3, minlat : float=52.2, maxlon: float=13.7, maxlat: float=52.3): - async with await psycopg.AsyncConnection.connect(conn_string) as conn: - async with conn.cursor() as cur: - # Download file directly - response.headers["Content-Disposition"] = f'attachment; filename="{type_name.value}.geojson"' - response.headers["Content-Type"] = 'application/geo+json' - - statement = sql.SQL("SELECT * FROM {table_name} (( SELECT * FROM ST_SetSRID(ST_MakeEnvelope(%s, %s, %s, %s), 4326) ));").format(table_name=sql.Identifier(type_name.name)) - await cur.execute(statement, ( minlon, minlat, maxlon, maxlat) ) - result = await cur.fetchone() - return result[0] - -@app.get("/boundaries/") -async def export_boundaries(response: Response, ids: Annotated[list[int], Query()]): - async with await psycopg.AsyncConnection.connect(conn_string, row_factory=dict_row) as conn: - async with conn.cursor() as cur: - # Check if ids are available - statement = sql.SQL("SELECT osm_id FROM boundaries WHERE osm_id = ANY(%s)") - await cur.execute(statement, (ids, )) - results = await cur.fetchall() - if results == None or len(results) != len(ids): - raise HTTPException(status_code=404, detail="Couldn't find given ids. At least one id is wrong or dupplicated.") - - response.headers["Content-Disposition"] = 'attachment; filename="boundaries_'+ '_'.join(map(str, ids)) +'.geojson"' - response.headers["Content-Type"] = 'application/geo+json' - - statement = sql.SQL("SELECT ST_AsGeoJSON(ST_Transform(ST_UNION(geom), 4326))::jsonb AS geom FROM boundaries WHERE osm_id = ANY(%s)") - await cur.execute(statement, (ids, ) ) - result = await cur.fetchone() - - return result['geom'] - - -@app.get("/verify/{type_name}/{osm_id}") -async def retrieve_verify_status(response: Response, type_name: VerificationTable, osm_id: int): - - async with await psycopg.AsyncConnection.connect(conn_string, row_factory=dict_row) as conn: - async with conn.cursor() as cur: - # Check if osm_id is available - statement = sql.SQL("SELECT * FROM {table_name} WHERE osm_id = %s ORDER BY verified_at DESC LIMIT 1").format(table_name=sql.Identifier(type_name.name)) - await cur.execute(statement, (osm_id,)) - - results = await cur.fetchone() - if results == None: - statement = sql.SQL("SELECT * FROM {table_name} WHERE osm_id = %s;").format(table_name=sql.Identifier(type_name.value)) - await cur.execute(statement, (osm_id,)) - results = await cur.fetchone() - if results == None: - raise HTTPException(status_code=404, detail="osm_id not found") - return { - "osm_id": osm_id, - "verified": "none" - } - return results - - -@app.get("/verify/{type_name}/{osm_id}/history") -async def retrieve_verify_history(response: Response, type_name: VerificationTable, osm_id: int): - async with await psycopg.AsyncConnection.connect(conn_string, row_factory=dict_row) as conn: - async with conn.cursor() as cur: - # Check if osm_id is available - statement = sql.SQL("SELECT * FROM {table_name} WHERE osm_id = %s ORDER BY verified_at DESC").format(table_name=sql.Identifier(type_name.name)) - await cur.execute(statement, (osm_id,)) - - results = await cur.fetchall() - if results == None: - raise HTTPException(status_code=404, detail="osm_id not found") - - return results - - -# TODO: guard `osm_type` and `verified_at` -@app.post("/verify/{type_name}/{osm_id}") -async def verify_osm_object(response: Response, type_name: VerificationTable, osm_type: str, osm_id: int, verified_at: str, verified_status: VerifiedState, verified_by: int=None, comment: str=''): - async with await psycopg.AsyncConnection.connect(conn_string) as conn: - async with conn.cursor() as cur: - # Check if osm_id is available - statement = sql.SQL("SELECT osm_id FROM {table_name} l WHERE l.osm_id = %s").format(table_name=sql.Identifier(type_name.value)) - await cur.execute(statement, (osm_id,)) - - results = await cur.fetchone() - if results == None: - raise HTTPException(status_code=404, detail="osm_id not found") - - statement = sql.SQL("INSERT INTO {table_name} (osm_type, osm_id, verified_at, verified_by, verified, comment) VALUES (%s, %s, %s, %s, %s, %s)").format(table_name=sql.Identifier(type_name.name)) - await cur.execute(statement, (osm_type, osm_id, verified_at, verified_by, verified_status.name, comment)) - await conn.commit() - - return 'OK' - -@app.get("/init") -async def init_api(response: Response, secret: str): - if secret != api_secret: - raise HTTPException(status_code=401, detail="the API secret is wrong. Access denied!") - async with await psycopg.AsyncConnection.connect(conn_string) as conn: - async with conn.cursor() as cur: - print("Starting creation of verification tables") - - sql_views_path = Path(__file__).with_name('INIT_VERIFICATION_VIEWS.sql') - with open(sql_views_path, "r") as f: - sql = f.read() - for table in VerificationTable: - print('Create verification table and view ', table.value) - processed_sql = sql.format( - verification_table="BikelaneVerification", # table.name, - geometry_table="bikelanes", # table.value, - joined_table="bikelanes_verified", # verified_table(table.value), - ) - await cur.execute(processed_sql) - await conn.commit() - - print('=' * 80) - print("Finished creation of verification tables") - - print("Starting creation of database exports") - sql_functions_path = Path(__file__).with_name('INIT_FUNCTIONS.sql') - with open(sql_functions_path, "r") as f: - sql = f.read() - for table in ExportTable: - function_name = table.name - print('Create function', function_name, ' for table ', table.value) - processed_sql = sql.replace('{function_name}', function_name).replace('{table_name}', table.value) - await cur.execute(processed_sql) - await conn.commit() - - print('=' * 80) - print("Finished creation of database exports") - print('=' * 80) - print("Finished database initialization") - - -@app.get("/health") -async def retrieve_service_health(): - async with await psycopg.AsyncConnection.connect(conn_string) as conn: - async with conn.cursor() as cur: - try: - await cur.execute(sql.SQL("SELECT 1")) - except: - raise HTTPException(status_code=500, detail="DB Connection is dead") - else: - return "OK" diff --git a/api/requirements.txt b/api/requirements.txt deleted file mode 100644 index 9811d164..00000000 --- a/api/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -fastapi -uvicorn -psycopg diff --git a/app/run-7-analysis.sh b/app/run-6-analysis.sh similarity index 100% rename from app/run-7-analysis.sh rename to app/run-6-analysis.sh diff --git a/app/run-6-api.sh b/app/run-6-api.sh deleted file mode 100755 index 3f048338..00000000 --- a/app/run-6-api.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -echo -e "\e[1m\e[7m API – START \e[27m\e[21m\e[0m" - -echo -e "INFO: Call api/init to create verification tables and export functions" -curl -G -d "secret=$API_SECRET" api/init --silent --output '/dev/null' - -echo -e "\e[1m\e[7m API – END \e[27m\e[21m\e[0m" diff --git a/app/run-8-metadata.sh b/app/run-7-metadata.sh similarity index 100% rename from app/run-8-metadata.sh rename to app/run-7-metadata.sh diff --git a/app/run.sh b/app/run.sh index 10f445bc..476bbe21 100755 --- a/app/run.sh +++ b/app/run.sh @@ -46,17 +46,13 @@ process_end_time=$(date +%s) export PROCESS_RUN_TIME_DIFF=$((process_end_time - process_start_time)) # used by metadata.sh if ! ./run-5-postprocess.sh; then - alert '*ERROR*: #run-6-postprocess exited with non-zero status code' + alert '*ERROR*: #run-5-postprocess exited with non-zero status code' fi -if ! ./run-6-api.sh; then - alert '*ERROR*: #run-7-api exited with non-zero status code' +if ! ./run-6-analysis.sh; then + alert '*ERROR*: #run-6-analysis exited with non-zero status code' fi -if ! ./run-7-analysis.sh; then - alert '*ERROR*: #run-5-analysis exited with non-zero status code' -fi - -if ! ./run-8-metadata.sh; then - alert '*ERROR*: #run-8-metadata exited with non-zero status code' +if ! ./run-7-metadata.sh; then + alert '*ERROR*: #run-7-metadata exited with non-zero status code' fi diff --git a/docker-compose.development.yml b/docker-compose.development.yml index fe3e1409..64d6afef 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -39,22 +39,6 @@ services: depends_on: db: condition: service_healthy - api: - build: - context: . - dockerfile: ./api.Dockerfile - container_name: api - environment: - PGHOST: - PGDATABASE: - PGUSER: - PGPASSWORD: - API_SECRET: - ports: - - 80:80 - depends_on: - db: - condition: service_healthy db: image: postgis/postgis:14-3.3-alpine shm_size: 1gb diff --git a/docker-compose.yml b/docker-compose.yml index 85ad13c2..cfab7e8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - 3333:3000 labels: - "traefik.enable=true" - - "traefik.http.routers.tiles.rule=Host(`${TILES_URL}`)" + - "traefik.http.routers.tiles.rule=Host(`${URL}`)" - "traefik.http.routers.tiles.entrypoints=websecure" - "traefik.http.routers.tiles.tls.certresolver=letsencrypt" - "traefik.http.routers.tiles.tls=true" @@ -46,29 +46,6 @@ services: depends_on: db: condition: service_healthy - api: - image: public.ecr.aws/n0p8j4k5/atlas/api:${GITHUB_SHA} - container_name: api - environment: - PGHOST: - PGDATABASE: - PGUSER: - PGPASSWORD: - API_SECRET: - restart: unless-stopped - ports: - - "8020:80" - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.rule=Host(`${API_URL}`)" - - "traefik.http.routers.api.entrypoints=websecure" - - "traefik.http.routers.api.tls.certresolver=letsencrypt" - - "traefik.http.routers.api.tls=true" - networks: - - app_bridge - depends_on: - db: - condition: service_healthy db: image: postgis/postgis:14-3.3-alpine container_name: db From 3785efd5523cb05eb4139004200de46a3a4b101d Mon Sep 17 00:00:00 2001 From: rush42 Date: Mon, 15 Jan 2024 16:41:16 +0100 Subject: [PATCH 4/6] remove unused helper functions --- app/process/helper/CheckDataWithinYears.lua | 66 ------------------- app/process/helper/HasAreaTags.lua | 54 --------------- .../__tests__/CheckDateWithinYears.test.lua | 55 ---------------- app/process/shared/IsFresh.lua | 44 ------------- app/process/shared/__test__/IsFresh.test.lua | 56 ---------------- 5 files changed, 275 deletions(-) delete mode 100644 app/process/helper/CheckDataWithinYears.lua delete mode 100644 app/process/helper/HasAreaTags.lua delete mode 100644 app/process/helper/__tests__/CheckDateWithinYears.test.lua delete mode 100644 app/process/shared/IsFresh.lua delete mode 100644 app/process/shared/__test__/IsFresh.test.lua diff --git a/app/process/helper/CheckDataWithinYears.lua b/app/process/helper/CheckDataWithinYears.lua deleted file mode 100644 index be5438e9..00000000 --- a/app/process/helper/CheckDataWithinYears.lua +++ /dev/null @@ -1,66 +0,0 @@ -function CheckDate(object, tag) - if(object.tags.is_present == true) then - -- (0) Only handle cases where our main data is present - if(object.tags['check_date:tag']) then - -- (1) If check_date is present, use it - local withinYears = CheckDataWithinYears(object.tags['check_date:tag'], 2) - if (withinYears.result) then - object.tags.fresh = 'fresh_check_date' - object.tags.fresh_age_days = withinYears.diffDays - object.tags._freshNotes = 'check_date used; fresh=true; confidence=high' - else - object.tags.fresh = 'outdated_check_date' - object.tags.fresh_age_days = withinYears.diffDays - object.tags._freshNotes = 'check_date used; fresh=false; confidence=high' - end - else - -- (2) Fall back to object's last update date - local withinYears = CheckDataWithinYears(os.date('!%Y-%m-%d', object.timestamp), 2) - if(withinYears.result) then - object.tags.fresh = 'fresh_update_at' - object.tags.fresh_age_days = withinYears.diffDays - object.tags._freshNotes = 'update_at used; fresh=true; confidence=low' - else - object.tags.fresh = 'outdated_update_at' - object.tags.fresh_age_days = withinYears.diffDays - object.tags._freshNotes = 'update_at used; fresh=false; confidence=low' - end - end - else - object.tags._freshNotes = 'is_present=false, so fresh data skipped' - end -end - - --- This helper takes a few shortcuts. But since precision is not that important, those should be fine. --- - We ignore the check_date-_day_ since that could be missing ("2022-01" is a valid value in OSM when exact day is unkown). --- - Minimum data of check_date is "yyyy-mm" --- - We can only compare full years (for now) --- - Default is 2 years; the exact behavior on the edges is not explitly coded/decided - --- * @desc CheckDataWithinYears(dateValue, [_opt_ 2 (default)]) --- * @returns `{ result = boolean, diffDays = int | nil }` -function CheckDataWithinYears(dateValue, years) - local yearsToSubtract = years or 2 - local compareDays = yearsToSubtract * 365 - - if not dateValue then - return { result = false } - end - - -- `2022-01` - -- For simplicty, we skip the days. - local checkDatePattern = "(%d+)-(%d+).*" - local checkDateYear, checkDateMonth, _rest = dateValue:match(checkDatePattern) - if not checkDateYear and not checkDateMonth then return { result = false } end - - local checkDateUnixtime = os.time({ year = checkDateYear, month = checkDateMonth, day = 1, hour = 12, min = 0, sec = 0 }) - - local today = os.date("*t") -- format: table (object) - local currentUnixtime = os.time({ year = today.year, month = today.month, day = today.day, hour = 12, min = 0, sec = 0 }) - - local diffTime = currentUnixtime - checkDateUnixtime - local diffDays = math.ceil(diffTime / 60 / 60 / 24) - - return { result = diffDays < compareDays, diffDays = diffDays } -end diff --git a/app/process/helper/HasAreaTags.lua b/app/process/helper/HasAreaTags.lua deleted file mode 100644 index e6709e6e..00000000 --- a/app/process/helper/HasAreaTags.lua +++ /dev/null @@ -1,54 +0,0 @@ --- TODO Lars uses a shorter version of what Jochen proposes in Github. --- However I am missing the check for "is_closed_line" in Jochens version… - --- Lars: -function HasAreaTags(tags) - if tags.area == 'yes' then - return true - end - if tags.area == 'no' then - return false - end - - return false -end - --- Jochen: https://github.com/openstreetmap/osm2pgsql/blob/master/flex-config/generic.lua#L184-L221 - --- Helper function that looks at the tags and decides if this is possibly an area. --- function has_area_tags(tags) --- if tags.area == 'yes' then --- return true --- end --- if tags.area == 'no' then --- return false --- end - --- return tags.aeroway --- or tags.amenity --- or tags.building --- or tags.harbour --- or tags.historic --- or tags.landuse --- or tags.leisure --- or tags.man_made --- or tags.military --- or tags.natural --- or tags.office --- or tags.place --- or tags.power --- or tags.public_transport --- or tags.shop --- or tags.sport --- or tags.tourism --- or tags.water --- or tags.waterway --- or tags.wetland --- or tags['abandoned:aeroway'] --- or tags['abandoned:amenity'] --- or tags['abandoned:building'] --- or tags['abandoned:landuse'] --- or tags['abandoned:power'] --- or tags['area:highway'] --- or tags['building:part'] --- end diff --git a/app/process/helper/__tests__/CheckDateWithinYears.test.lua b/app/process/helper/__tests__/CheckDateWithinYears.test.lua deleted file mode 100644 index a6a96f16..00000000 --- a/app/process/helper/__tests__/CheckDateWithinYears.test.lua +++ /dev/null @@ -1,55 +0,0 @@ -package.path = package.path .. ";app/process/helper/?.lua" -require("CheckDataWithinYears") -require("PrintTable") - -local debug = false -print('=== Test CheckDataWithinYears ===') -if (debug) then print('=== Debug true ===') end - --- Format: https://www.lua.org/pil/22.1.html -local today = os.date("*t") - -local desc = -"check_date yesterday returns the number of days of this month because we always compare with the beginning of the month" -local yesterday = os.date("%Y-%m-%d", os.time({ year = today.year, month = today.month, day = tonumber(today.day) - 1 })) -local withinYears = CheckDataWithinYears(yesterday) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == true) -assert(withinYears.diffDays == today.day - 1) - -local desc = -"check_date last year is within range, diffDays somewhere > 300 (the excat values depents on the current month" -local lastYear = os.date("%Y-%m-%d", os.time({ year = tonumber(today.year) - 1, month = today.month, day = today.day })) -local withinYears = CheckDataWithinYears(lastYear) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == true) -assert(withinYears.diffDays > 300) - -local desc = "check_date 2 year is (already/just) outside of our range" -local twoYears = os.date("%Y-%m-%d", os.time({ year = tonumber(today.year) - 2, month = today.month, day = today.day })) -local withinYears = CheckDataWithinYears(twoYears) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == false) - -local desc = "check_date 3 year is outside of our range" -local threeYears = os.date("%Y-%m-%d", os.time({ year = tonumber(today.year) - 3, month = today.month, day = today.day })) -local withinYears = CheckDataWithinYears(threeYears) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == false) - -local desc = "Just a year will return `false` every time" -local withinYears = CheckDataWithinYears(tostring(tonumber(today.year) - 1)) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == false) - -local desc = "No check_date will return `false` every time" -local withinYears = CheckDataWithinYears(nil) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == false) - -local desc = "check_date 3 year compared to 4 years is within range" -local threeYears = os.date("%Y-%m-%d", os.time({ year = tonumber(today.year) - 3, month = today.month, day = today.day })) -local withinYears = CheckDataWithinYears(threeYears, 4) -if (debug) then PrintTableWithHeadline(withinYears, desc) end -assert(withinYears.result == true) -assert(withinYears.diffDays > 365 * 3) diff --git a/app/process/shared/IsFresh.lua b/app/process/shared/IsFresh.lua deleted file mode 100644 index d607f11e..00000000 --- a/app/process/shared/IsFresh.lua +++ /dev/null @@ -1,44 +0,0 @@ -package.path = package.path .. ";./app/process/helper/?.lua" -require("CheckDataWithinYears") - --- * @desc Categoize the fresshnews of data. --- If present, use `date_tag` (eg. `check_date:lit`) => high confidence. --- Fall back to `object.timestamp` => low confidence. --- * @returns --- `dest` with freshness-tags added to it. --- If no parameter was provided a new object gets created. --- `isFreshKey` a table of keys to use for FilterTags -function IsFresh(object, date_tag, dest, prefix) - dest = dest or {} - - local date = os.date('!%Y-%m-%d', object.timestamp) - local source = "update_at" - - if object.tags[date_tag] then - date = object.tags[date_tag] - source = "check_date" - end - - -- Freshness of data, see documentation - local withinYears = CheckDataWithinYears(date, 2) - - local fresh_key = "fresh" - if (prefix) then - fresh_key = table.concat({ prefix, "fresh" }, "_") - end - if withinYears.result then - dest[fresh_key] = "fresh_" .. source - else - dest[fresh_key] = "outdated_" .. source - end - - local fresh_age_key = "fresh_age_days" - if (prefix) then - fresh_age_key = table.concat({ prefix, "fresh_age_days" }, "_") - end - dest[fresh_age_key] = withinYears.diffDays - - local isFreshKeys = { fresh_key, fresh_age_key } - - return dest, isFreshKeys -end diff --git a/app/process/shared/__test__/IsFresh.test.lua b/app/process/shared/__test__/IsFresh.test.lua deleted file mode 100644 index 16a37539..00000000 --- a/app/process/shared/__test__/IsFresh.test.lua +++ /dev/null @@ -1,56 +0,0 @@ -package.path = package.path .. ";./app/process/helper/?.lua;./app/process/shared/?.lua" -require("IsFresh") -require("PrintTable") - -local daySeconds = 24 * 60 * 60 -local tenYearsSeconds = daySeconds * 365 * 10 -local dateTimeTenYearsAgo = os.time() - tenYearsSeconds -local formattedDateTenYearsAgo = os.date('!%Y-%m-%d', os.time() - tenYearsSeconds) -local osmObject = { - ["timestamp"] = dateTimeTenYearsAgo, - ["tags"] = { ["check_date:lit"] = tostring(formattedDateTenYearsAgo) } -} - -print('=== Test IsFresh: keys ===') -local _dest, keys = IsFresh(osmObject, "check_date:lit", {}) --- PrintTableWithHeadline(keys, 'keys') -assert(keys[1] == "fresh") -assert(keys[2] == "fresh_age_days") - -print('=== Test IsFresh: keys + postif ===') -local _dest, keys = IsFresh(osmObject, "check_date:lit", {}, 'foo') --- PrintTableWithHeadline(keys, 'keys') -assert(keys[1] == "foo_fresh") -assert(keys[2] == "foo_fresh_age_days") - -print('=== Test IsFresh: outdated_check_date ===') -local dest, _keys = IsFresh(osmObject, "check_date:lit", {}) --- PrintTableWithHeadline(dest, 'dest') -assert(dest["fresh"] == "outdated_check_date") -assert(dest["fresh_age_days"] > 3600) - -print('=== Test IsFresh: outdated_check_date + postfix ===') -local dest, _keys = IsFresh(osmObject, "check_date:lit", {}, 'foo') --- PrintTableWithHeadline(dest, 'dest') -assert(dest["foo_fresh"] == "outdated_check_date") -assert(dest["foo_fresh_age_days"] > 3600) - -print('=== Test IsFresh: fresh_check_date ===') -local yesterday = os.date('!%Y-%m-%d', os.time() - daySeconds) -osmObject["tags"]["check_date:lit"] = tostring(yesterday) -local dest, _keys = IsFresh(osmObject, "check_date:lit", {}) --- PrintTableWithHeadline(dest, 'dest') -assert(dest["fresh"] == "fresh_check_date") - -print('=== Test IsFresh: outdated_update_at ===') -local yesterday = os.date('!%Y-%m-%d', os.time() - (daySeconds * 365 * 2 + 10)) -osmObject["tags"]["check_date:lit"] = nil -local dest, _keys = IsFresh(osmObject, "check_date:lit", {}) --- PrintTableWithHeadline(dest, 'dest') -assert(dest["fresh"] == "outdated_update_at") - -print('=== Test IsFresh: fresh_update_at ===') -osmObject["timestamp"] = os.time() - (daySeconds * 10) -local dest, _keys = IsFresh(osmObject, "check_date:lit", {}) --- PrintTableWithHeadline(dest, 'dest') -assert(dest["fresh"] == "fresh_update_at") From 10ac953a894392449e7922cb73ddd255c0d703d3 Mon Sep 17 00:00:00 2001 From: rush42 Date: Thu, 25 Jan 2024 11:48:13 +0100 Subject: [PATCH 5/6] us `ln -f` instead of `mv`, to make `--timestamping` work again --- app/run-1-download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/run-1-download.sh b/app/run-1-download.sh index 00f3e557..c51abb02 100755 --- a/app/run-1-download.sh +++ b/app/run-1-download.sh @@ -17,7 +17,7 @@ fi echo "Downloading file: ${OSM_DOWNLOAD_URL}" # Note: Showing the progress (locally) is very verbose, unfortunately if wget --timestamping --no-verbose ${OSM_DOWNLOAD_URL} --directory-prefix=${OSM_DATADIR}; then - mv ${OSM_DOWNLOAD_FILE} ${OSM_LOCAL_FILE} + ln -f ${OSM_DOWNLOAD_FILE} ${OSM_LOCAL_FILE} else echo "Error: Failed to download the file from ${OSM_DOWNLOAD_URL}" fi From 103af8dfaa4b888a8e32c9650b72ffc01ab96792 Mon Sep 17 00:00:00 2001 From: rush42 Date: Thu, 25 Jan 2024 12:40:15 +0100 Subject: [PATCH 6/6] rename `boundariesLabels` -> `boundaryLabels` --- app/process/boundaries/boundaries.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/process/boundaries/boundaries.lua b/app/process/boundaries/boundaries.lua index f28ddcc0..4accf76a 100644 --- a/app/process/boundaries/boundaries.lua +++ b/app/process/boundaries/boundaries.lua @@ -14,8 +14,8 @@ local table = osm2pgsql.define_table({ } }) -local tableLabel = osm2pgsql.define_table({ - name = 'boundariesLabel', +local labelTable = osm2pgsql.define_table({ + name = 'boundaryLabels', ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, columns = { { column = 'tags', type = 'jsonb' }, @@ -76,7 +76,7 @@ function osm2pgsql.process_relation(object) meta = Metadata(object), geom = object:as_multipolygon() }) - tableLabel:insert({ + labelTable:insert({ tags = results_tags, meta = Metadata(object), geom = object:as_multipolygon():centroid()