Skip to content

Commit

Permalink
fix: Improved JSON parser, producews queries that work with a local p…
Browse files Browse the repository at this point in the history 
…ostgres
  • Loading branch information
@rsavoye
rsavoye committed Sep 3, 2023
1 parent 21bbd9b commit 0ca40d9
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 149 deletions.
176 changes: 63 additions & 113 deletions osm_rawdata/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
# import time
from pathlib import Path
from shapely.geometry import shape
import flatdict


# Find the other files for this project
Expand All @@ -53,14 +54,17 @@ def __init__(self,
self.config = {'select': dict(),
'tables': list(),
'where': dict(),
'keep': list()
'keep': list(),
}
self.config['select'] = {'nodes': [], 'ways_poly': [], 'ways_line': []}
self.config['where'] = {'nodes': [], 'ways_poly': [], 'ways_line': []}
self.config['select'] = {'nodes': [],
'ways_poly': [],
'ways_line': [],
}
self.config['where'] = {'nodes': [],
'ways_poly': [],
'ways_line': [],
}
self.geometry = boundary
# These are only in the JSON queries used for Export Tool
self.outputtype = None
self.filename = None
# for polygon extracts, sometimes we just want the center point
self.centroid = False

Expand Down Expand Up @@ -136,101 +140,50 @@ def parseJson(self,
data = json.load(file)
# Get the geometry
self.geometry = shape(data['geometry'])

# This is only used by Export Tool
if 'outputType' in data:
self.outputtype = data['outputType']

# This is only used by Export Tool
if 'fileName' in data:
self.filename = data['fileName']

# This is only used by Export Tool
if 'geometryType' not in data:
data['geometryType'] = 'all_geometry'

# Get the list of tables to query
for table in data['geometryType']:
if table.lower() == 'all_geometry':
self.config['tables'].append('ways_poly')
self.config['tables'].append('ways_line')
self.config['tables'].append('nodes')
# self.config['tables'].append('relations')
elif table.lower() == 'line':
self.config['tables'].append('ways_line')
elif table.lower() == 'polygon':
self.config['tables'].append('ways_poly')
elif table.lower() == 'point':
self.config['tables'].append('nodes')

if 'filters' not in data:
data['filters'] = {}
# The filter define the tags to be used.
for k, v in data['filters'].items():
if k == 'tags':
if 'point' in v:
for k1, v1 in v['point'].items():
for k2, v2 in v1.items():
tag = dict()
self.config['select']['nodes'].append({k2: v2})
if k1[:4] == 'join':
tag['op'] = k1[5:]
tag[k2] = v2
self.config['where']['nodes'].append(tag)
# print(f"POINT: {k2} == {v2}")
elif 'line' in v:
for k1, v1 in v['line'].items():
# print(f"LINE: {k} = {v1}")
self.config['where']['ways_line'].append(v1)
self.config['select']['ways_line'].append(v1)
elif 'polygon' in v:
for k1, v1 in v['polgon'].items():
# print(f"POLY: {k} = {v1}")
self.config['select']['ways_poly'].append(v1)
self.config['where']['ways_poly'].append(v1)
elif 'all_geometry' in v:
# import epdb ; epdb.st()
for k1, v1 in v['all_geometry'].items():
print(f"ALL_GEOMETRY: {k1} == {v1}")
self.config['select']['nodes'].append(v1)
self.config['select']['ways_poly'].append(v1)
self.config['select']['ways_line'].append(v1)
# Where is the same tags, but has a or/and
# if k1[:4] == 'join':
# v1['op'] = k1[5:]
self.config['where']['nodes'].append(v1)
self.config['where']['ways_poly'].append(v1)
self.config['where']['ways_line'].append(v1)
# Anything under attributes scans the values that aren't
# part of the data, Tags like osm_id, version, uid, user,
# and timestamp.
if k == 'attributes':
# print(f"FIXME: {k} = {v}")
if 'all_geometry' in v:
print(f"ALL_GEOMETRY2 : {k} == {v}")
for k1 in v['all_geometry']:
tag = {k1: []}
self.config['select']['nodes'].append(tag)
self.config['select']['ways_line'].append(tag)
self.config['select']['ways_poly'].append(tag)
else:
if type(v) == dict:
if 'point' in v:
for v1 in v['point']:
# print(f"POINT2: {v1}")
self.config['select']['nodes'].append({v1: []})
# else:
# print(f"OOPS: {v1}")
if 'line' in v:
for v1 in v['line']:
# print(f"LINE2: {v1}")
self.config['select']['ways_line'].append({v1: []})
if 'polygon' in v:
for v1 in v['polygon']:
self.config['select']['ways_poly'].append({v1: []})
# print(f"POLY2: {v1}")
for key, value in flatdict.FlatDict(data).items():
keys = key.split(':')
# print(keys)
# print(f"\t{value}")
# We already have the geometry
if key[:8] == 'geometry':
continue
if len(keys) == 1:
self.config.update({key: value})
continue
# keys[0] is currently always 'filters'
# keys[1] is currently 'tags' for the WHERE clause,
# of attributes for the SELECT
geom = keys[2]
# tag = keys[4]
# Get the geometry
if geom == 'point':
geom = 'nodes'
elif geom == 'line':
geom = 'ways_line'
elif geom == 'polygon':
geom = 'ways_poly'
if keys[1] == 'attributes':
for v1 in value:
if geom == 'all_geometry':
self.config['select']['nodes'].append({v1: {}})
self.config['select']['ways_line'].append({v1: {}})
self.config['select']['ways_poly'].append({v1: {}})
self.config['tables'].append('nodes')
self.config['tables'].append('ways_poly')
self.config['tables'].append('ways_line')
else:
self.config['select'].append({v: []})
self.config['tables'].append(geom)
self.config['select'][geom].append({v1: {}})
if keys[1] == 'tags':
newtag = {keys[4]: value}
newtag['op'] = keys[3][5:]
if geom == 'all_geometry':
self.config['where']['nodes'].append(newtag)
self.config['where']['ways_poly'].append(newtag)
self.config['where']['ways_line'].append(newtag)
else:
self.config['where'][geom].append(newtag)

return self.config

def dump(self):
Expand All @@ -240,10 +193,10 @@ def dump(self):
print("Dumping QueryConfig class")

# These two data items are only used by Export Tool for output files
if self.filename:
print(f"The output filename is {self.filename}")
if self.outputtype:
print(f"The output type is {self.outputtype}")
# for k, v in self.config.items():
# if k == 'nodes' or k == 'ways_poly' or k == 'ways_line' or k == 'keep' or k == 'tables' k ==:
# continue
# print(f"Other {k} is \'{v}\'")

keys = list()
for key, value in self.config['select'].items():
Expand All @@ -263,25 +216,22 @@ def dump(self):
for key, value in self.config['where'].items():
if type(value) == list:
for v in value:
if 'op' not in v:
for k, v in v.items():
for v1 in v:
print(f"\tWhere table \'{key}\': {k} has value \'{v1}\'")
continue
op = v['op'].upper()
del v['op']
# del v['op']
if type(v) == str:
print(f"\tWhere table \'{key}\' has value \'{v}\'")
keys.append(v)
continue
for k1, v1 in v.items():
keys.append(v1)
if k1 == 'op':
continue
print(f"\tWhere table \'{key}\', tag \'{k1}\' has values \'{v1}\' {op}")
else:
print(f"\tSelecting tag \'{key}\'")
print("Tables")
for table in self.config['tables']:
print(f"\t{table}")
#print("Tables")
#for table in self.config['tables']:
# print(f"\t{table}")
if self.geometry:
print(self.geometry)

Expand Down
77 changes: 41 additions & 36 deletions osm_rawdata/db/postgres.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,15 @@
import argparse
import logging
import sys
import os
import json
from sys import argv
from geojson import Point, Feature, FeatureCollection, dump, Polygon
from geojson import Feature, FeatureCollection, dump, Polygon
import geojson
import requests
import time
import psycopg2
from shapely.geometry import shape, Polygon
import shapely
import subprocess
from pathlib import Path
from osm_rawdata.config import QueryConfig
from shapely import wkt
Expand Down Expand Up @@ -251,44 +250,51 @@ def createSQL(self,
join_and = list()
for entry in config.config['where'][table]:
# print(entry)
if 'op' not in entry:
pass
op = entry['op']
del entry['op']
[[k, v]] = entry.items()
if op == 'or':
print(f"1: {k} OR {v}")
join_or.append(entry)
elif op == 'and':
print(f"2: {k} AND {v}")
join_and.append(entry)

for k, v in entry.items():
if k == 'op':
continue
if op == 'or':
# print(f"1: {k}=\'{v}\' OR ")
join_or.append(entry)
elif op == 'and':
# print(f"2: {k}=\'{v}\' AND ")
join_and.append(entry)
# jor = '('
jor = ''
# FIXME: v is now a list
for entry in join_or:
[[k, v]] = entry.items()
for v1 in v:
jor += f"tags->>\'{k}\'=\'{v1}\' OR "
# jor = f"{jor[:-4]})"
jor = f"{jor[:-4]}"
print(f"JOR: {jor}")
if len(join_and) > 0:
# jand = ' AND ('
jand = ' AND '
# else:
# jand = '('
for k, v in entry.items():
if k == 'op':
continue
if len(v) == 1:
v1 = f"=\'{v[0]}\'"
elif len(v) > 0:
v1 = f" IN {str(tuple(v))}"
else:
v1 = 'IS NOT NULL'
jor += f"tags->>\'{k}\' {v1} OR "
#print(f"JOR: {jor}")

jand = ''
for entry in join_and:
[[k, v]] = entry.items()
for v1 in v:
jand += f"tags->>\'{k}\'=\'{v1}\' AND "
# jand = f"{jand[:-4]})"
jand = f"{jand[:-4]}"
print(f"JAND: {jand}")

query = f"{select} FROM {table} WHERE {jor} {jand}"
print(query)
sql.append(query)
for k, v in entry.items():
if k == 'op':
continue
if len(v) == 1:
v1 = f"=\'{v[0]}\'"
elif len(v) > 0:
v1 = f" IN {str(tuple(v))}"
else:
v1 = 'IS NOT NULL AND'
jand += f"tags->>\'{k}\' {v1} AND "
#print(f"JAND: {jand}")
query = f"{select} FROM {table} WHERE {jor} {jand}".rstrip()
#if query[len(query)-5:] == ' OR ':
# print(query[:query.rfind(' ')])
sql.append(query[:query.rfind(' ')])

return sql

def queryLocal(self,
Expand Down Expand Up @@ -326,7 +332,7 @@ def queryLocal(self,
elif query.find(" relations ") > 0:
query = query.replace("relations", "relations_view")
features = list()
log.debug(query)
# log.debug(query)
self.dbcursor.execute(query)
result = self.dbcursor.fetchall()
log.info("Query returned %d records" % len(result))
Expand All @@ -349,7 +355,6 @@ def queryLocal(self,
if item[i] is not None:
tags[k] = item[i]
i += 1
print(f"TAGS: {tags}")
else:
# Figure out the tags from the custom SELECT
end = query.find('FROM')
Expand Down

0 comments on commit 0ca40d9

Please sign in to comment.