add deps to pyproject toml (#5)

* add deps to pyproject toml * fixes for codespell hook * resolve ruff errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * pass ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
worldbank · Jun 25, 2024 · 01e0c93 · 01e0c93
1 parent 040cda6
commit 01e0c93
Show file tree

Hide file tree

Showing 33 changed files with 554 additions and 1,067 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -49,4 +49,4 @@ jobs:
           name: python-package-distributions
           path: dist/
       - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,6 +29,7 @@ repos:
       - id: codespell
         name: codespell
         description: Checks for common misspellings in text files
+        additional_dependencies: [tomli]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.1
     hooks:

diff --git a/notebooks/AWS_Summarize.ipynb b/notebooks/AWS_Summarize.ipynb
@@ -15,14 +15,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import sys, os, importlib\n",
-    "import rasterio, geojson, h3, boto3\n",
+    "import boto3\n",
     "\n",
-    "import pandas as pd\n",
-    "import geopandas as gpd\n",
-    "\n",
-    "from botocore.config import Config\n",
-    "from botocore import UNSIGNED"
+    "import pandas as pd"
    ]
   },
   {
@@ -106,31 +101,35 @@
     }
    ],
    "source": [
-    "bucket = 'wbg-geography01' \n",
-    "prefix = 'sylvera'\n",
-    "region = 'us-east-1'\n",
-    "s3client = boto3.client('s3', region_name=region)\n",
+    "bucket = \"wbg-geography01\"\n",
+    "prefix = \"sylvera\"\n",
+    "region = \"us-east-1\"\n",
+    "s3client = boto3.client(\"s3\", region_name=region)\n",
     "\n",
     "# Loop through the S3 bucket and get all the file keys\n",
     "more_results = True\n",
     "try:\n",
-    "    del(token)\n",
-    "except:\n",
+    "    del token  # noqa\n",
+    "except Exception:\n",
     "    pass\n",
     "loops = 0\n",
     "\n",
     "all_res = []\n",
     "while more_results:\n",
     "    print(f\"Completed loop: {loops}\")\n",
     "    if loops > 0:\n",
-    "        objects = s3client.list_objects_v2(Bucket=bucket, ContinuationToken=token, Prefix=prefix)\n",
+    "        objects = s3client.list_objects_v2(\n",
+    "            Bucket=bucket,\n",
+    "            ContinuationToken=token,  # noqa\n",
+    "            Prefix=prefix,  # noqa\n",
+    "        )\n",
     "    else:\n",
     "        objects = s3client.list_objects_v2(Bucket=bucket)\n",
-    "    more_results = objects['IsTruncated']\n",
+    "    more_results = objects[\"IsTruncated\"]\n",
     "    if more_results:\n",
-    "        token = objects['NextContinuationToken']\n",
+    "        token = objects[\"NextContinuationToken\"]\n",
     "    loops += 1\n",
-    "    for res in objects['Contents']:\n",
+    "    for res in objects[\"Contents\"]:\n",
     "        all_res.append(res)"
    ]
   },
@@ -311,7 +310,7 @@
    ],
    "source": [
     "inD = pd.DataFrame(all_res)\n",
-    "inD['folder'] = inD['Key'].apply(lambda x: \"_\".join(x.split(\"/\")[:])"
+    "inD[\"folder\"] = inD[\"Key\"].apply(lambda x: \"_\".join(x.split(\"/\")[:]))"
    ]
   },
   {

diff --git a/notebooks/BUILT_compare_buildings_wsf.ipynb b/notebooks/BUILT_compare_buildings_wsf.ipynb
@@ -15,19 +15,18 @@
     }
    ],
    "source": [
-    "import sys, os\n",
+    "import sys\n",
+    "import os\n",
     "import rasterio\n",
     "\n",
     "import pandas as pd\n",
     "import geopandas as gpd\n",
-    "import numpy as np\n",
     "\n",
     "from shapely.wkt import loads\n",
     "\n",
     "sys.path.insert(0, \"../src\")\n",
     "\n",
     "import GOSTrocks.rasterMisc as rMisc\n",
-    "import GOSTrocks.dataMisc as dMisc\n",
     "\n",
     "%load_ext autoreload\n",
     "%autoreload 2"
@@ -49,19 +48,25 @@
    ],
    "source": [
     "# Local/input files\n",
-    "iso3 = 'KHM'\n",
+    "iso3 = \"KHM\"\n",
     "out_folder = \"c:/WBG/Work/KHM_Energy/data\"\n",
     "wsf_file = os.path.join(out_folder, \"WSF\", \"wsf.tif\")\n",
     "ghsl_file = os.path.join(out_folder, \"GHSL\", \"ghsl.tif\")\n",
-    "overture_buildings = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29.csv\")\n",
-    "overture_raster = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29.tif\")\n",
-    "overture_raster_points = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29_points.tif\")\n",
+    "overture_buildings = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29.csv\"\n",
+    ")\n",
+    "overture_raster = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29.tif\"\n",
+    ")\n",
+    "overture_raster_points = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29_points.tif\"\n",
+    ")\n",
     "for file in [wsf_file, ghsl_file]:\n",
     "    if not os.path.exists(os.path.dirname(file)):\n",
     "        os.makedirs(os.path.dirname(file))\n",
     "\n",
     "# get country extent from geopandas\n",
-    "world_filepath = gpd.datasets.get_path('naturalearth_lowres')\n",
+    "world_filepath = gpd.datasets.get_path(\"naturalearth_lowres\")\n",
     "world = gpd.read_file(world_filepath)\n",
     "country = world[world.iso_a3 == iso3]"
    ]
@@ -97,12 +102,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#Clip GHSL using local files\n",
-    "local_version = r\"J:\\Data\\GLOBAL\\GHSL\\Built\\GHS_BUILT_S_E2020_GLOBE_R2023A_54009_100_V1_0.tif\"\n",
+    "# Clip GHSL using local files\n",
+    "local_version = (\n",
+    "    r\"J:\\Data\\GLOBAL\\GHSL\\Built\\GHS_BUILT_S_E2020_GLOBE_R2023A_54009_100_V1_0.tif\"\n",
+    ")\n",
     "if not os.path.exists(ghsl_file):\n",
     "    ghsl_raster = rasterio.open(local_version)\n",
     "    data, profile = rMisc.clipRaster(ghsl_raster, country)\n",
-    "    with rasterio.open(ghsl_file, 'w', **profile) as dst:\n",
+    "    with rasterio.open(ghsl_file, \"w\", **profile) as dst:\n",
     "        dst.write(data)\n",
     "ghsl_r = rasterio.open(ghsl_file)"
    ]
@@ -203,9 +210,9 @@
    "source": [
     "# read in and process Overture buildings\n",
     "ob = pd.read_csv(overture_buildings)\n",
-    "ob_geoms = ob['wkt'].apply(loads)\n",
+    "ob_geoms = ob[\"wkt\"].apply(loads)\n",
     "inB = gpd.GeoDataFrame(ob, geometry=ob_geoms, crs=4326)\n",
-    "inB.head()\n"
+    "inB.head()"
    ]
   },
   {
@@ -216,9 +223,11 @@
    "source": [
     "# attempt to rasterrize the buildings as polygons\n",
     "if not os.path.exists(overture_raster):\n",
-    "    rasterized_buildings = rMisc.rasterizeDataFrame(inB, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.)\n",
-    "    with rasterio.open(overture_raster, 'w', **rasterized_buildings['meta']) as dst:\n",
-    "        dst.write_band(1, rasterized_buildings['vals'])\n",
+    "    rasterized_buildings = rMisc.rasterizeDataFrame(\n",
+    "        inB, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.0\n",
+    "    )\n",
+    "    with rasterio.open(overture_raster, \"w\", **rasterized_buildings[\"meta\"]) as dst:\n",
+    "        dst.write_band(1, rasterized_buildings[\"vals\"])\n",
     "overture_r = rasterio.open(overture_raster)"
    ]
   },
@@ -231,11 +240,15 @@
     "# attempt to rasterrize the buildings as points\n",
     "if not os.path.exists(overture_raster_points):\n",
     "    inB_points = inB.copy()\n",
-    "    inB_points['geometry'] = inB_points['geometry'].centroid\n",
-    "    rasterized_buildings = rMisc.rasterizeDataFrame(inB_points, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.)\n",
-    "    with rasterio.open(overture_raster_points, 'w', **rasterized_buildings['meta']) as dst:\n",
-    "        dst.write_band(1, rasterized_buildings['vals'])\n",
-    "overture_r_points = rasterio.open(overture_raster_points) "
+    "    inB_points[\"geometry\"] = inB_points[\"geometry\"].centroid\n",
+    "    rasterized_buildings = rMisc.rasterizeDataFrame(\n",
+    "        inB_points, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.0\n",
+    "    )\n",
+    "    with rasterio.open(\n",
+    "        overture_raster_points, \"w\", **rasterized_buildings[\"meta\"]\n",
+    "    ) as dst:\n",
+    "        dst.write_band(1, rasterized_buildings[\"vals\"])\n",
+    "overture_r_points = rasterio.open(overture_raster_points)"
    ]
   },
   {
@@ -250,10 +263,10 @@
     "ghsl_thresh = 3000\n",
     "\n",
     "o_data = overture_r_points.read(1)\n",
-    "o_data = (o_data > o_thresh).astype('uint8')\n",
+    "o_data = (o_data > o_thresh).astype(\"uint8\")\n",
     "\n",
     "ghsl_data = ghsl_r.read(1)\n",
-    "ghsl_data = (ghsl_data > ghsl_thresh).astype('uint8') * 10\n",
+    "ghsl_data = (ghsl_data > ghsl_thresh).astype(\"uint8\") * 10\n",
     "\n",
     "combo_data = o_data + ghsl_data\n",
     "\n",
@@ -262,8 +275,8 @@
     "if not os.path.exists(out_file):\n",
     "    meta = overture_r_points.meta.copy()\n",
     "    meta.update(dtype=rasterio.uint8, nodata=0)\n",
-    "    with rasterio.open(out_file, 'w', **meta) as out_raster:\n",
-    "        out_raster.write_band(1, combo_data)\n"
+    "    with rasterio.open(out_file, \"w\", **meta) as out_raster:\n",
+    "        out_raster.write_band(1, combo_data)"
    ]
   },
   {

diff --git a/notebooks/FATHOM/CLIP_Flood_data_iso3.ipynb b/notebooks/FATHOM/CLIP_Flood_data_iso3.ipynb
@@ -19,7 +19,6 @@
    "source": [
     "import sys\n",
     "import os\n",
-    "import boto3\n",
     "import rasterio\n",
     "\n",
     "import geopandas as gpd\n",
@@ -63,11 +62,11 @@
     "out_folder = f\"/home/wb411133/temp/FATHOM/{iso3}\"\n",
     "if not os.path.exists(out_folder):\n",
     "    os.makedirs(out_folder)\n",
-    "    \n",
+    "\n",
     "# This demo uses the default national boundaries included with GeoPandas, but this can be changed here\n",
-    "world_filepath = gpd.datasets.get_path('naturalearth_lowres')\n",
+    "world_filepath = gpd.datasets.get_path(\"naturalearth_lowres\")\n",
     "world = gpd.read_file(world_filepath)\n",
-    "inB = world.loc[world['iso_a3'] == iso3].copy()"
+    "inB = world.loc[world[\"iso_a3\"] == iso3].copy()"
    ]
   },
   {
@@ -77,18 +76,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Select layer to downlaod\n",
-    "flood_type = [\"COASTAL\",\"FLUVIAL\",\"PLUVIAL\"]\n",
+    "# Select layer to download\n",
+    "flood_type = [\"COASTAL\", \"FLUVIAL\", \"PLUVIAL\"]\n",
     "defence = [\"DEFENDED\"]\n",
-    "return_period = ['1in5','1in10','1in50']\n",
+    "return_period = [\"1in5\", \"1in10\", \"1in50\"]\n",
     "climate_model = [\"PERCENTILE50\"]\n",
     "year = [\"2020\"]\n",
     "\n",
     "# all_vrts is a pandas dataframe with all the vrt paths to the global datasets, with columns defining\n",
     "# the various models' defining attributes\n",
     "all_vrts = dMisc.get_fathom_vrts(True)\n",
-    "sel_images = all_vrts.loc[(all_vrts['FLOOD_TYPE'].isin(flood_type)) & (all_vrts['DEFENCE'].isin(defence)) & \n",
-    "             (all_vrts['RETURN'].isin(return_period))  & (all_vrts['CLIMATE_MODEL'].isin(climate_model))]"
+    "sel_images = all_vrts.loc[\n",
+    "    (all_vrts[\"FLOOD_TYPE\"].isin(flood_type))\n",
+    "    & (all_vrts[\"DEFENCE\"].isin(defence))\n",
+    "    & (all_vrts[\"RETURN\"].isin(return_period))\n",
+    "    & (all_vrts[\"CLIMATE_MODEL\"].isin(climate_model))\n",
+    "]"
    ]
   },
   {
@@ -349,12 +352,12 @@
     "# For each image in the selected images dataframe, we clip out the area of interest\n",
     "#     which is defined by the ioso3 code, but could be any GeoDataFrame\n",
     "\n",
-    "for idx, row in sel_images.iterrows():    \n",
-    "    out_file = os.path.join(out_folder, os.path.basename(row['PATH']))\n",
+    "for idx, row in sel_images.iterrows():\n",
+    "    out_file = os.path.join(out_folder, os.path.basename(row[\"PATH\"]))\n",
     "    if not os.path.exists(out_file):\n",
-    "        cur_r = rasterio.open(row['PATH'])\n",
+    "        cur_r = rasterio.open(row[\"PATH\"])\n",
     "        rMisc.clipRaster(cur_r, inB, out_file)\n",
-    "    tPrint(os.path.basename(row['PATH']))"
+    "    tPrint(os.path.basename(row[\"PATH\"]))"
    ]
   }
  ],

diff --git a/notebooks/FATHOM/PROCESSING_NOTEBOOKS/Generate_footprints.ipynb b/notebooks/FATHOM/PROCESSING_NOTEBOOKS/Generate_footprints.ipynb
@@ -148,12 +148,12 @@
     "\n",
     "# Add a boolean flag in the fluvial catalog if there is a corresponding coastal tile\n",
     "fluvial_tiles[\"COASTAL\"] = 0\n",
-    "for idx, row in tqdm(xx.iterrows()):\n",
+    "for idx, row in tqdm(xx.iterrows()):  # noqa\n",
     "    if row[\"ID\"] in all_files:\n",
     "        fluvial_tiles.loc[idx, \"COASTAL\"] = 1\n",
     "\n",
     "# There are ~300 coastal tiles that don't have a corresponding fluvial tile\n",
-    "coastal_missing = [x for x in all_files if x not in xx[\"ID\"].values]\n",
+    "coastal_missing = [x for x in all_files if x not in xx[\"ID\"].values]  # noqa\n",
     "len(coastal_missing)"
    ]
   },