diff --git a/api/app/googleflood.py b/api/app/googleflood.py index 4a96e1667..b3599710d 100644 --- a/api/app/googleflood.py +++ b/api/app/googleflood.py @@ -1,12 +1,17 @@ """Get data from Humanitarian Data Cube (HDC) API""" import logging -from os import getenv - +import os import requests +import geopandas as gpd +import pandas as pd +import uuid + +from fiona.drvsupport import supported_drivers +supported_drivers['LIBKML'] = 'rw' logger = logging.getLogger(__name__) -GOOGLE_FLOODS_API_KEY = getenv("GOOGLE_FLOODS_API_KEY", "") +GOOGLE_FLOODS_API_KEY = os.getenv("GOOGLE_FLOODS_API_KEY", "") if GOOGLE_FLOODS_API_KEY == "": logger.warning("Missing backend parameter: GOOGLE_FLOODS_API_KEY") @@ -20,6 +25,8 @@ def format_to_geojson(data): "properties": { "gaugeId": data["gaugeId"], "issuedTime": data["issuedTime"], + # not present on all data tested at this point + # handle in the future # "forecastTimeRange": data["forecastTimeRange"], # "forecastChange": data["forecastChange"], # "forecastTrend": data["forecastTrend"], @@ -38,7 +45,6 @@ def get_google_floods_gauges( asGeojson: bool = True, ): """Get statistical charts data""" - logging.info(GOOGLE_FLOODS_API_KEY) URL = f'https://floodforecasting.googleapis.com/v1/floodStatus:searchLatestFloodStatusByArea?key={GOOGLE_FLOODS_API_KEY}' response = requests.post( @@ -55,3 +61,45 @@ def get_google_floods_gauges( return response +from pydantic import BaseModel +from typing import List + +class InundationMap(BaseModel): + level: str + serializedPolygonId: str + +class InundationMapSet(BaseModel): + inundationMaps: List[InundationMap] + + +def get_google_floods_inundations( + inundationMapSet: InundationMapSet, +) -> gpd.GeoDataFrame: + """Get statistical charts data""" + level_to_kml = dict() + URL = 'https://floodforecasting.googleapis.com/v1/serializedPolygons/{serializedPolygonId}?key={key}' + for inundationMap in inundationMapSet: + response = requests.get( + URL.format( + serializedPolygonId=inundationMap['serializedPolygonId'], + key = GOOGLE_FLOODS_API_KEY + ) + ).json() + level_to_kml[inundationMap['level']] = response['kml'] + + # Create a temp path for writing kmls + tmp_path = os.path.join(f'/tmp/google-floods/{str(uuid.uuid4())}') + if not os.path.exists(tmp_path): + os.makedirs(tmp_path) + + gdf_buff = [] + for level, kml in level_to_kml.items(): + kml_path = os.path.join(tmp_path, f'{level}.kml') + with open(kml_path, 'w') as f: + f.write(kml) + gdf = gpd.read_file(kml_path, driver='KML') + gdf['level'] = level + gdf_buff.append(gdf) + + gdf = pd.concat(gdf_buff) + return gdf diff --git a/api/app/main.py b/api/app/main.py index 96b84b021..ff0e89ad7 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -14,7 +14,7 @@ from app.database.alert_model import AlchemyEncoder, AlertModel from app.database.database import AlertsDataBase from app.database.user_info_model import UserInfoModel -from app.googleflood import get_google_floods_gauges +from app.googleflood import get_google_floods_gauges, get_google_floods_inundations from app.hdc import get_hdc_stats from app.kobo import get_form_dates, get_form_responses, parse_datetime_params from app.models import AcledRequest, RasterGeotiffModel @@ -419,4 +419,12 @@ def post_raster_geotiff(raster_geotiff: RasterGeotiffModel): def get_google_floods_gauges_api( iso2: str, ): - return get_google_floods_gauges(iso2) \ No newline at end of file + return get_google_floods_gauges(iso2) + + +@app.get('/google-floods-inundations') +def get_google_floods_inundations_api( + inundationMapSet_JSONString: str, +): + """Get statistical charts data""" + return get_google_floods_inundations(json.loads(inundationMapSet_JSONString)).to_json() diff --git a/api/app/tests/test_google_floods_api.py b/api/app/tests/test_google_floods_api.py index a6335a539..d0ad5973f 100644 --- a/api/app/tests/test_google_floods_api.py +++ b/api/app/tests/test_google_floods_api.py @@ -1,20 +1,32 @@ -from app.googleflood import get_google_floods_gauges +from app.googleflood import get_google_floods_gauges, get_google_floods_inundations from app.main import app from fastapi.testclient import TestClient import logging, json client = TestClient(app) +test_inundation_map_set = [{'level': 'HIGH', 'serializedPolygonId': '9631a3143f2543ae9664c4fba7298931'}, {'level': 'MEDIUM', 'serializedPolygonId': '93444f067f42409493fd907b2b0323b7'}, {'level': 'LOW', 'serializedPolygonId': '757ac60c06ea481aba384201a107d611'}] +test_inundation_map_set_jsonstr = json.dumps(test_inundation_map_set) + def test_get_google_floods_gauges(): gauges = get_google_floods_gauges("BD") assert len(gauges) > 0 +def test_get_google_floods_inundations(): + floods = get_google_floods_inundations(test_inundation_map_set) + assert floods.shape[0] > 0 + + def test_get_google_floods_gauges_api(): response = client.get("/google-floods-gauges?iso2=BD") assert response.status_code == 200 - logging.info(json.dumps(response.json())) - # logging.info(response.json()) assert len(response.json()) > 0 - \ No newline at end of file + +def test_get_google_floods_inundation_api(): + response = client.get(f"/google-floods-inundations?inundationMapSet_JSONString={test_inundation_map_set_jsonstr}") + assert response.status_code == 200 + logging.info(json.dumps(response.json())) + # logging.info(response.json()) + assert len(response.json()) > 0 \ No newline at end of file diff --git a/api/poetry.lock b/api/poetry.lock index 711cb7c19..4706e0f0b 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "affine" @@ -604,6 +604,29 @@ smb = ["smbprotocol"] ssh = ["paramiko"] tqdm = ["tqdm"] +[[package]] +name = "geopandas" +version = "1.0.1" +description = "Geographic pandas extensions" +optional = false +python-versions = ">=3.9" +files = [ + {file = "geopandas-1.0.1-py3-none-any.whl", hash = "sha256:01e147d9420cc374d26f51fc23716ac307f32b49406e4bd8462c07e82ed1d3d6"}, + {file = "geopandas-1.0.1.tar.gz", hash = "sha256:b8bf70a5534588205b7a56646e2082fb1de9a03599651b3d80c99ea4c2ca08ab"}, +] + +[package.dependencies] +numpy = ">=1.22" +packaging = "*" +pandas = ">=1.4.0" +pyogrio = ">=0.7.2" +pyproj = ">=3.3.0" +shapely = ">=2.0.0" + +[package.extras] +all = ["GeoAlchemy2", "SQLAlchemy (>=1.3)", "folium", "geopy", "mapclassify", "matplotlib (>=3.5.0)", "psycopg-binary (>=3.1.0)", "pyarrow (>=8.0.0)", "xyzservices"] +dev = ["black", "codecov", "pre-commit", "pytest (>=3.1.0)", "pytest-cov", "pytest-xdist"] + [[package]] name = "graphql-core" version = "3.2.3" @@ -891,24 +914,6 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] -[[package]] -name = "importlib-resources" -version = "6.1.0" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_resources-6.1.0-py3-none-any.whl", hash = "sha256:aa50258bbfa56d4e33fbd8aa3ef48ded10d1735f11532b8df95388cc6bdb7e83"}, - {file = "importlib_resources-6.1.0.tar.gz", hash = "sha256:9d48dcccc213325e810fd723e7fbb45ccb39f6cf5c31f00cf2b965f5f10f3cb9"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] - [[package]] name = "iniconfig" version = "2.0.0" @@ -961,9 +966,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} jsonschema-specifications = ">=2023.03.6" -pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -983,7 +986,6 @@ files = [ ] [package.dependencies] -importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} referencing = ">=0.28.0" [[package]] @@ -1412,17 +1414,6 @@ files = [ {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] -[[package]] -name = "pkgutil-resolve-name" -version = "1.3.10" -description = "Resolve a name to an object." -optional = false -python-versions = ">=3.6" -files = [ - {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, - {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, -] - [[package]] name = "platformdirs" version = "3.11.0" @@ -1457,7 +1448,6 @@ files = [ [package.dependencies] greenlet = "2.0.2" pyee = "9.0.4" -typing-extensions = {version = "*", markers = "python_version <= \"3.8\""} [[package]] name = "pluggy" @@ -1580,6 +1570,52 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1)", "mypy", "pre-commit docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] +[[package]] +name = "pyogrio" +version = "0.9.0" +description = "Vectorized spatial vector file format I/O using GDAL/OGR" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyogrio-0.9.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:1a495ca4fb77c69595747dd688f8f17bb7d2ea9cd86603aa71c7fc98cc8b4174"}, + {file = "pyogrio-0.9.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:6dc94a67163218581c7df275223488ac9b31dc582ccd756da607c3338908566c"}, + {file = "pyogrio-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e38c3c6d37cf2cc969407e4d051dcb507cfd948eb26c7b0840c4f7d7d4a71bd4"}, + {file = "pyogrio-0.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f47c9b6818cc0f420015b672d5dcc488530a5ee63e5ba35a184957b21ea3922a"}, + {file = "pyogrio-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb04bd80964428491951766452f0071b0bc37c7d38c45ef02502dbd83e5d74a0"}, + {file = "pyogrio-0.9.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f5d80eb846be4fc4e642cbedc1ed0c143e8d241653382ecc76a7620bbd2a5c3a"}, + {file = "pyogrio-0.9.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:2f2ec57ab74785db9c2bf47c0a6731e5175595a13f8253f06fa84136adb310a9"}, + {file = "pyogrio-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a289584da6df7ca318947301fe0ba9177e7f863f63110e087c80ac5f3658de8"}, + {file = "pyogrio-0.9.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:13642608a1cd67797ae8b5d792b0518d8ef3eb76506c8232ab5eaa1ea1159dff"}, + {file = "pyogrio-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:9440466c0211ac81f3417f274da5903f15546b486f76b2f290e74a56aaf0e737"}, + {file = "pyogrio-0.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2e98913fa183f7597c609e774820a149e9329fd2a0f8d33978252fbd00ae87e6"}, + {file = "pyogrio-0.9.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f8bf193269ea9d347ac3ddada960a59f1ab2e4a5c009be95dc70e6505346b2fc"}, + {file = "pyogrio-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f964002d445521ad5b8e732a6b5ef0e2d2be7fe566768e5075c1d71398da64a"}, + {file = "pyogrio-0.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:083351b258b3e08b6c6085dac560bd321b68de5cb4a66229095da68d5f3d696b"}, + {file = "pyogrio-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:796e4f6a4e769b2eb6fea9a10546ea4bdee16182d1e29802b4d6349363c3c1d7"}, + {file = "pyogrio-0.9.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:7fcafed24371fe6e23bcf5abebbb29269f8d79915f1dd818ac85453657ea714a"}, + {file = "pyogrio-0.9.0-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:30cbeeaedb9bced7012487e7438919aa0c7dfba18ac3d4315182b46eb3139b9d"}, + {file = "pyogrio-0.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4da0b9deb380bd9a200fee13182c4f95b02b4c554c923e2e0032f32aaf1439ed"}, + {file = "pyogrio-0.9.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:4e0f90a6c3771ee1f1fea857778b4b6a1b64000d851b819f435f9091b3c38c60"}, + {file = "pyogrio-0.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:959022f3ad04053f8072dc9a2ad110c46edd9e4f92352061ba835fc91df3ca96"}, + {file = "pyogrio-0.9.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:2829615cf58b1b24a9f96fea42abedaa1a800dd351c67374cc2f6341138608f3"}, + {file = "pyogrio-0.9.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:17420febc17651876d5140b54b24749aa751d482b5f9ef6267b8053e6e962876"}, + {file = "pyogrio-0.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a2fcaa269031dbbc8ebd91243c6452c5d267d6df939c008ab7533413c9cf92d"}, + {file = "pyogrio-0.9.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:019731a856a9abfe909e86f50eb13f8362f6742337caf757c54b7c8acfe75b89"}, + {file = "pyogrio-0.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:d668cb10f2bf6ccd7c402f91e8b06290722dd09dbe265ae95b2c13db29ebeba0"}, + {file = "pyogrio-0.9.0.tar.gz", hash = "sha256:6a6fa2e8cf95b3d4a7c0fac48bce6e5037579e28d3eb33b53349d6e11f15e5a8"}, +] + +[package.dependencies] +certifi = "*" +numpy = "*" +packaging = "*" + +[package.extras] +benchmark = ["pytest-benchmark"] +dev = ["Cython"] +geopandas = ["geopandas"] +test = ["pytest", "pytest-cov"] + [[package]] name = "pyparsing" version = "3.1.1" @@ -1668,7 +1704,6 @@ files = [ ] [package.dependencies] -importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""} jsonschema = {version = ">=4.18,<5.0", optional = true, markers = "extra == \"validation\""} python-dateutil = ">=2.7.0" @@ -3020,5 +3055,5 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-it [metadata] lock-version = "2.0" -python-versions = "<3.11,>=3.8.0" -content-hash = "e9224e9bd4ff2cd4a0b981b7331f45962d31448e297eb747faffada8032ec181" +python-versions = "<3.11,>=3.9.0" +content-hash = "174175156ddd8041448f74b555795573eaf0796fdaf27d45af5b343d56e5f60a" diff --git a/api/pyproject.toml b/api/pyproject.toml index ca34d2673..c8eb2d024 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -7,7 +7,7 @@ license = "Apache 2.0" [tool.poetry.dependencies] PyJWT = "2.4.0" -python = "<3.11,>=3.8.0" +python = "<3.11,>=3.9.0" requests = "^2.32.0" playwright = "1.38.0" fastapi = "^0.109.1" @@ -28,6 +28,7 @@ pystac = "^1.8.4" pytest-asyncio = "^0.21.1" pytest-playwright = "^0.4.3" rasterio = "^1.3.9" +geopandas = "^1.0.1" [tool.poetry.group.dev.dependencies] black = "^24.3.0"