Skip to content

Commit

Permalink
Merge remote-tracking branch 'local/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
@robsavoye
robsavoye committed Aug 29, 2023
2 parents 93dc3af + c269a69 commit 1893946
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 85 deletions.
136 changes: 86 additions & 50 deletions osm_rawdata/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ def __init__(self,
Args:
boundary (Polygon): The project boundary
"""
self.config = {'select': list(),
'tables': list(),
'where': list(),
'keep': list()
self.config = {'select': dict(),
'tables': list(),
'where': dict(),
'keep': list()
}
self.config['select'] = {'nodes': [], 'ways_poly': [], 'ways_line': []}
self.config['where'] = {'nodes': [], 'ways_poly': [], 'ways_line': []}
self.geometry = boundary
# These are only in the JSON queries used for Export Tool
self.outputtype = None
Expand Down Expand Up @@ -166,40 +168,73 @@ def parseJson(self,
data['filters'] = {}
# The filter define the tags to be used.
for k, v in data['filters'].items():
if k == 'tags' and 'all_geometry' in v:
for k1, v1 in v['all_geometry'].items():
if k1 == 'join_or':
if k == 'tags':
if 'point' in v:
for k1, v1 in v['point'].items():
for k2, v2 in v1.items():
if type(v2) == list and len(v2) == 0:
tag = {k2: v2, 'op': 'or'}
self.config['where'].append(tag)
else:
v2['op'] = 'or'
self.config['where'].append(v2)
elif k1 == 'join_and':
v1['op'] = 'and'
self.config['where'].append(v1)
self.config['select']['nodes'].append({k2: v2})
self.config['where']['nodes'].append({k2: v2})
# print(f"POINT: {k2} == {v2}")
elif 'line' in v:
for k1, v1 in v['line'].items():
# print(f"LINE: {k} = {v1}")
self.config['where']['ways_line'].append(v1)
self.config['select']['ways_line'].append(v1)
elif 'polygon' in v:
for k1, v1 in v['polgon'].items():
# print(f"POLY: {k} = {v1}")
self.config['select']['ways_poly'].append(v1)
self.config['where']['ways_poly'].append(v1)
elif 'all_geometry' in v:
# import epdb ; epdb.st()
for k1, v1 in v['all_geometry'].items():
# print(f"ALL_GEOMETRY: {k1} == {v1}")
if k1[:4] == 'join':
v1['op'] = k1[5:]
self.config['select']['nodes'].append(v1)
self.config['select']['ways_poly'].append(v1)
self.config['select']['ways_line'].append(v1)
# Where is the same tags, but has a or/and
self.config['where']['nodes'].append(v1)
self.config['where']['ways_poly'].append(v1)
self.config['where']['ways_line'].append(v1)
# Anything under attributes scans the values that aren't
# part of the data, Tags like osm_id, version, uid, user,
# and timestamp.
if k == 'attributes':
# print(f"FIXME: {k} = {v}")
if 'all_geometry' in v:
for i in v['all_geometry']:
self.config['select'].append({i: []})
self.config['select']['nodes'].append({i: []})
self.config['select']['ways_line'].append({i: []})
self.config['select']['ways_poly'].append({i: []})
else:
print(v)
# FIXME: this is a hack
if type(v) == dict:
continue
if 'point' in v:
for v1 in v['point']:
# print(f"POINT2: {v1}")
self.config['select']['nodes'].append(v1)
# else:
# print(f"OOPS: {v1}")

if 'line' in v:
for v1 in v['line']:
# print(f"LINE2: {v1}")
self.config['select']['ways_line'].append(v1)
if 'polygon' in v:
for v1 in v['polygon']:
self.config['select']['ways_poly'].append(v1)
# print(f"POLY2: {v1}")
else:
self.config['select'].append({v: []})

for entry in self.config['where']:
for k, v in entry.items():
if k == 'op':
continue
# print(f"bar: {k} = {v}")
self.config['select'].append({k: v})
# for entry in self.config['where']:
# for k, v in entry.items():
# if k == 'op':
# continue
# print(f"bar: {k} = {v}")
# #for k1, v1 in v
# self.config['select'].append({k: v})

return self.config

Expand All @@ -215,32 +250,33 @@ def dump(self):
if self.outputtype:
print(f"The output type is {self.outputtype}")

print("Select: ")
for entry in self.config['select']:
if type(entry) == dict:
[[k, v]] = entry.items()
if len(v) > 0:
print(f"\tSelecting tag \'{k}\' is \'{v}\'")
else:
print(f"\tSelecting tag \'{k}\'")
keys = list()
for key, value in self.config['select'].items():
if type(value) == list:
for v in value:
if type(v) == str:
print(f"\tSelecting tag \'{key}\' has value \'{v}\'")
keys.append(v)
continue
for k1, v1 in v.items():
keys.append(v1)
print(f"\tSelecting tag \'{key}\' \'{k1}\' has values \'{v1}\'")
else:
print(f"\tSelecting tag \'{key}\'")
#print(f"\tSelecting tag \'{key}\' \'{k1}\' has values \'{keys}\'")
print("Where: ")
for entry in self.config['where']:
if type(entry) == dict:
# op = entry['op']
for k, v in entry.items():
if k != 'op':
if type(v) == list and len(v) == 0:
print(f"\twhere tag \'{k}\' is \'not null\'")
else:
print(f"\twhere tag \'{k}\' is \'{v}\'")
else:
if entry['op'] is not None:
print(f"\t{entry['op']}")
elif type(entry) == list:
for item in entry:
print(f"{item}")
for key, value in self.config['where'].items():
if type(value) == list:
for v in value:
if type(v) == str:
print(f"\tWhere tag \'{key}\' has value \'{v}\'")
keys.append(v)
continue
for k1, v1 in v.items():
keys.append(v1)
print(f"\tWhere tag \'{key}\' \'{k1}\' has values \'{v1}\'")
else:
print(f"\tReturn tag \'{entry}\'")
print(f"\tSelecting tag \'{key}\'")
print("Tables")
for table in self.config['tables']:
print(f"\t{table}")
Expand Down
1 change: 1 addition & 0 deletions osm_rawdata/db/postgres.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ def __init__(self,
Args:
dburi (str): The URI string for the database connection
Returns:
status (bool): Whether the data base connection was sucessful
"""
Expand Down
16 changes: 8 additions & 8 deletions tests/levels.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,22 +65,22 @@
"school",
"college",
"university",
"mosque ",
" church ",
" temple",
"mosque",
"church",
"temple",
"supermarket",
"marketplace",
"clinic",
"hospital",
"police",
"fire_station",
"stadium ",
" sports_centre",
"sports_centre",
"governor_office ",
" townhall ",
" subdistrict_office ",
" village_office ",
" community_group_office",
"townhall ",
"subdistrict_office ",
"village_office ",
"community_group_office",
"government_office"
],
"man_made": [
Expand Down
66 changes: 39 additions & 27 deletions tests/test_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Copyright (c) 2023 Humanitarian OpenStreetMap Team
#
# This file is part of osm_rawdata.
# This file is part of osm_fieldwork.
#
# This is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
Expand All @@ -15,67 +15,76 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with osm_rawdata. If not, see <https:#www.gnu.org/licenses/>.
# along with osm_fieldwork. If not, see <https:#www.gnu.org/licenses/>.
#
"""Test JSON functionality."""

import os
import sys
from osm_rawdata.config import QueryConfig

#
# The JSON data files came from the raw-data-api project, and are currently
# used by that project for testing.
#

# Find the other files for this project
import osm_rawdata as rw
from osm_rawdata.config import QueryConfig

rootdir = rw.__path__[0]
if os.path.basename(rootdir) == "osm_rawdata":
rootdir = "./tests/"
# print(f"\t{rootdir}")

if os.path.basename(rootdir) == 'osm_rawdata':
rootdir = f"./tests/"

def test_levels():
# this query contains only the geometry and the output file name and type
# this query contains many levels and geometries
qc = QueryConfig()
qc.parseJson(f"{rootdir}/levels.json")
qc.dump()
assert qc.filename == "Example export with all features" and qc.outputtype == "geojson"
data = qc.parseJson(f"{rootdir}/levels.json")
# qc.dump()
hits = 0
if data['select']['nodes'][0]['amenity'][0] == 'bank':
hits += 1

if data['select']['ways_line'][2] == 'waterway':
hits += 1

if data['select']['ways_poly'][2] == 'admin_level':
hits += 1

if qc.filename == "Example export with all features" and qc.outputtype == "geojson":
hits += 1

assert hits == 4

def test_filters():
# this query contains only the geometry and the output file name and type
qc = QueryConfig()
qc.parseJson(f"{rootdir}/filters.json")
qc.dump()
pp = qc.parseJson(f"{rootdir}/filters.json")
# qc.dump()
hits = 0
if "name" in qc.config["select"][0]:
hits += 1
if "addr" in qc.config["select"][1]:
if 'name' in qc.config['select']['nodes'][2]:
hits += 1
if "building" in qc.config["select"][2]:
if 'addr' in qc.config['select']['nodes'][3]:
hits += 1
if "cafe" in qc.config["select"][3]["amenity"]:
if 'building' in 'building' in qc.config['select']['nodes'][0]:
hits += 1
if "restaurant" in qc.config["select"][3]["amenity"]:
if 'cafe' in qc.config['select']['nodes'][1]['amenity']:
hits += 1
if "pub" in qc.config["select"][3]["amenity"]:
if 'restaurant' in qc.config['select']['ways_poly'][1]['amenity']:
hits += 1

assert hits == 6


assert hits == 5

def test_formats():
# this query contains only the geometry and the output file name and type
qc = QueryConfig()
qc.parseJson(f"{rootdir}/formats.json")
assert qc.outputtype == "shp" and qc.filename == "Pokhara_all_features"


def test_everything():
# this query contains only the geometry, we want everything within this polygon
qc = QueryConfig()
qc.parseJson(f"{rootdir}/everything.json")
data = "POLYGON ((83.96919250488281 28.194446860487773, 83.99751663208006 28.194446860487773, 83.99751663208006 28.214869548073377, 83.96919250488281 28.214869548073377, 83.96919250488281 28.194446860487773))"
assert data == str(qc.geometry)


if __name__ == "__main__":
print("--- test_everything() ---")
test_everything()
Expand All @@ -85,3 +94,6 @@ def test_everything():
test_levels()
print("--- test_filters ---")
test_filters()
print("--- done() ---")


0 comments on commit 1893946

Please sign in to comment.