Merge pull request #33 from carbonplan/s3_update

Updates Azure sources to s3

Author: norlandrhagen

offset-project-fire/buffer_pool_analysis.ipynb

@@ -34,12 +34,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import intake\n",
     "import numpy as np\n",
     "import pandas as pd\n",
-    "import intake\n",
-    "\n",
-    "from carbonplan_styles.mpl import set_theme\n",
     "from carbonplan_styles.colors import colors\n",
+    "from carbonplan_styles.mpl import set_theme\n",
     "\n",
     "# set options\n",
     "set_theme(style=\"carbonplan_light\")\n",
@@ -60,7 +59,7 @@
    "outputs": [],
    "source": [
     "df = pd.read_excel(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-articles/offset-project-fire/arboc_issuance.xlsx\",\n",
+    "    \"https://carbonplan-articles.s3.us-west-2.amazonaws.com/offset-project-fire/arboc_issuance.xlsx\",\n",
     "    sheet_name=3,\n",
     "    engine=\"openpyxl\",\n",
     ")\n",
@@ -100,11 +99,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = df[\n",
-    "    ~df[\"Forest Buffer Account Contribution\"].apply(\n",
-    "        lambda x: isinstance(x, str)\n",
-    "    )\n",
-    "]\n",
+    "df = df[~df[\"Forest Buffer Account Contribution\"].apply(lambda x: isinstance(x, str))]\n",
     "df = df[df[\"ARB Offset Credits Issued\"] > 0]\n",
     "forest_df = df[df[\"Project Type\"] == \"Forest\"]\n",
     "\n",
@@ -254,15 +249,11 @@
     "perc_credits[\"state\"] = perc_credits.index.map(proj_state_map)\n",
     "perc_credits = perc_credits.rename(\n",
     "    # clean up column title\n",
-    "    columns={\n",
-    "        \"ARB Offset Credits Issued\": \"Percent Forest ARB Offset Credits Issued\"\n",
-    "    }\n",
+    "    columns={\"ARB Offset Credits Issued\": \"Percent Forest ARB Offset Credits Issued\"}\n",
     ")\n",
     "\n",
     "proj_loc = perc_credits.index.get_loc(proj_id)\n",
-    "display(\n",
-    "    perc_credits[: proj_loc + 1]\n",
-    ")  # +1 to include ACR260, otherwise off by one"
+    "display(perc_credits[: proj_loc + 1])  # +1 to include ACR260, otherwise off by one"
    ]
   },
   {
@@ -298,15 +289,11 @@
     "sorted_credits[\"state\"] = sorted_credits.index.map(proj_state_map)\n",
     "sorted_credits = sorted_credits.rename(\n",
     "    # clean up column title\n",
-    "    columns={\n",
-    "        \"ARB Offset Credits Issued\": \"Percent Forest ARB Offset Credits Issued\"\n",
-    "    }\n",
+    "    columns={\"ARB Offset Credits Issued\": \"Percent Forest ARB Offset Credits Issued\"}\n",
     ")\n",
     "\n",
     "proj_loc = sorted_credits.index.get_loc(proj_id)\n",
-    "display(\n",
-    "    sorted_credits[: proj_loc + 1]\n",
-    ")  # +1 to include ACR260, otherwise off by one"
+    "display(sorted_credits[: proj_loc + 1])  # +1 to include ACR260, otherwise off by one"
    ]
   },
   {

offset-project-fire/catalog.yaml

@@ -24,7 +24,7 @@ sources:
         allowed: [noaa, suomi, modis]
     driver: geojson
     args:
-      urlpath: 'https://carbonplan.blob.core.windows.net/carbonplan-articles/offset-project-fire/lionshead-{{sensor}}.json'
+      urlpath: 'hhttps://carbonplan-articles.s3.us-west-2.amazonaws.com/offset-project-fire/lionshead-{{sensor}}.json'
 
   project:
     metadata:
@@ -41,7 +41,7 @@ sources:
           url: https://americancarbonregistry.org/
     driver: geojson
     args:
-      urlpath: 'https://carbonplan.blob.core.windows.net/carbonplan-articles/offset-project-fire/carb_project_geometry.json'
+      urlpath: 'https://carbonplan-articles.s3.us-west-2.amazonaws.com/offset-project-fire/carb_project_geometry.json'
 
   arboc_issuance:
     metadata:
@@ -58,4 +58,4 @@ sources:
           url: https://ww2.arb.ca.gov/
     driver: csv
     args:
-      urlpath: 'https://carbonplan.blob.core.windows.net/carbonplan-articles/offset-project-fire/arb_offset_credit_issuance.csv'
+      urlpath: 'https://carbonplan-articles.s3.us-west-2.amazonaws.com/offset-project-fire/arb_offset_credit_issuance.csv'

offset-project-fire/fire_analysis.ipynb

@@ -56,14 +56,14 @@
     "import copy\n",
     "import os\n",
     "\n",
-    "from carbonplan_styles.colors import colors\n",
     "import fsspec\n",
     "import geopandas\n",
     "import hvplot.pandas\n",
     "import intake\n",
     "import pandas as pd\n",
     "import pyproj\n",
     "import shapely\n",
+    "from carbonplan_styles.colors import colors\n",
     "from shapely.ops import cascaded_union\n",
     "\n",
     "# plot styles\n",
@@ -79,7 +79,7 @@
     "# open our data catalog\n",
     "cat = intake.open_catalog(\"./catalog.yaml\")\n",
     "\n",
-    "write_to_azure = False"
+    "write_to_s3 = False"
    ]
   },
   {
@@ -244,9 +244,7 @@
    "source": [
     "burned_poly = cascaded_union(full_res.geometry.values)\n",
     "burned_gdf = (\n",
-    "    geopandas.GeoDataFrame.from_dict(\n",
-    "        {\"row\": {\"geometry\": burned_poly}}, orient=\"index\"\n",
-    "    )\n",
+    "    geopandas.GeoDataFrame.from_dict({\"row\": {\"geometry\": burned_poly}}, orient=\"index\")\n",
     "    .set_geometry(\"geometry\")\n",
     "    .set_crs(working_crs)\n",
     ")"
@@ -310,10 +308,7 @@
     "\n",
     "burned_project_area = burned_project.area.sum()\n",
     "print(\"Burned Project Area: %.2f m2\" % burned_project_area)\n",
-    "print(\n",
-    "    \"Percent Project Burned: %.2f%%\"\n",
-    "    % (burned_project_area / project_area * 100)\n",
-    ")"
+    "print(\"Percent Project Burned: %.2f%%\" % (burned_project_area / project_area * 100))"
    ]
   },
   {
@@ -323,13 +318,10 @@
    "outputs": [],
    "source": [
     "# dump the polygons out to a geojson file for use in the article.\n",
-    "if write_to_azure:\n",
-    "    # blob file system\n",
-    "    from adlfs import AzureBlobFileSystem\n",
+    "if write_to_s3:\n",
+    "    import s3fs\n",
     "\n",
-    "    fs = AzureBlobFileSystem(\n",
-    "        account_name=\"carbonplan\", account_key=os.environ[\"BLOB_ACCOUNT_KEY\"]\n",
-    "    )\n",
+    "    fs = s3fs.s3fs.S3FileSystem()\n",
     "    with fs.open(\n",
     "        \"carbonplan-articles/offset-project-fire/lionshead_fire_polygons.json\",\n",
     "        \"w\",\n",