From 9115a46f2ce7e0f6665b5f3e10ec88b157650a98 Mon Sep 17 00:00:00 2001 From: Deena Sun Date: Mon, 18 Nov 2024 16:00:03 -0800 Subject: [PATCH 1/3] [test] hello world function --- api/webscraper/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/webscraper/database.py b/api/webscraper/database.py index d0c02b5..46170ff 100644 --- a/api/webscraper/database.py +++ b/api/webscraper/database.py @@ -29,6 +29,10 @@ geocodio = GeocodioClient(geocode_api) +def hello(): + print("hello world") + + def nyserda_large_to_database(): database = [] database.extend(query_nyserda_large()) From 29f0433e00fc68b2d012cb183d3d33b8e94068b3 Mon Sep 17 00:00:00 2001 From: Deena Sun Date: Fri, 22 Nov 2024 10:35:37 -0800 Subject: [PATCH 2/3] [feat] set up Flask backend and api reroutes in next.config.mjs --- api/app.py | 22 ++++++++ api/webscraper/__init__.py | 0 .../database_constants.cpython-312.pyc | Bin 1256 -> 1256 bytes .../__pycache__/nyiso_scraper.cpython-312.pyc | Bin 8913 -> 8918 bytes .../nyserda_scraper.cpython-312.pyc | Bin 7721 -> 7726 bytes api/webscraper/database.py | 17 +++--- api/webscraper/nyiso_scraper.py | 4 +- api/webscraper/nyserda_scraper.py | 4 +- api/webscraper/ores_scraper.py | 4 +- api/webscraper/server.py | 50 ++++++++++++++++++ next.config.mjs | 8 +++ 11 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 api/app.py create mode 100644 api/webscraper/__init__.py create mode 100644 api/webscraper/server.py diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000..bcbc87e --- /dev/null +++ b/api/app.py @@ -0,0 +1,22 @@ +from flask import Flask +from webscraper.database import * +from webscraper.nyserda_scraper import * + +app = Flask(__name__) + +# flask run --port 5328 --debug + + +@app.route("/api/hello", methods=["GET"]) +def hello_world(): + return "Hello, World!" + + +@app.route("/api/nyserda_large", methods=["GET"]) +async def run_nyserda_large(): + nyserda_large_to_database() + return {"message": "Hello NYSERDA Large"} + + +if __name__ == "__main__": + app.run(port=5328, debug=True) diff --git a/api/webscraper/__init__.py b/api/webscraper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/webscraper/__pycache__/database_constants.cpython-312.pyc b/api/webscraper/__pycache__/database_constants.cpython-312.pyc index abca84bc4b1426e03d1509412e16a7196df19ae2..e31f734811da8e26f0908b44ebe84dacd23ecee8 100644 GIT binary patch delta 20 acmaFC`GS-CG%qg~0}$-2v)jmhmjwVo5C!i5 delta 20 acmaFC`GS-CG%qg~0}#x&u-eFdmjwVlq6Lcp diff --git a/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc b/api/webscraper/__pycache__/nyiso_scraper.cpython-312.pyc index ccd35bc365e2f3dd0f2a9e18462542b536bd9f7c..64ccf93d86c23f60d1f2660f22ee0a1cb352e25d 100644 GIT binary patch delta 1061 zcmZXT%WD%s9LF=scJtgmvf1WUZ4xaQ!P2L%R@#QPkV}+Gp+q6wX4ZD=CY#P|DmjQ3 zK?M=QT)avTQWP(H@gVpI)Qbordl0V$4;}=e2X%g%mSWw*C*R-9Z@$0ZJo0hj^MbS| zNg)sap6$)9)3!7}|<;W`pHXct|#?nJiIflDA72Lzwk9#@exQ{b|qnt@BbEa^NGu=XYWgnK7^2qTQ zT9!tO*Z0BBzKNcDQPTeIdn}-c-3&ZMObneKkMTL;19WjXe#@+2y=oB=BH^yqqK>FG z8d@Fa{X~Jhezp`U$_bbcZ#6a2s?=LLHA%sK6Z(lzitR-HpaR>G#?o$*e2?gs)=8>x z`%P-rNy+{p4GH2I?sA~8Z%APy(R00n;GY3x0XbHS&UncP+l@}2ABB-Iz&Kz6Fv+n< zE`q!SxD2?$dgOSq4lK#AAr15fF+JyktF$#Be<4sF*r$b6|lmQ7fGB24`1>6ByA?(=1L#37A9cirTE{9SG6Y`T?7Z0`L~_&84LVG# zL~ZbsbfeB|j$+kP*Ho%is%D*9YTcrphwWRV6T}aTeNAEz=*bZp>bgdvuB2wOK@5;F soqfp0CcC-r?5O~%1c1vWs{jo^0rxp}JtPG>!Ia!E8fNpkCjt!o1-K~dfB*mh delta 1019 zcmZXT&rcIU6vsR5c3avm(3Y~@wvo2jLl2+=BH#}!0U`%L6BD!^nuS@R+F#5rHRa;P zgGMgRT>J|RMlNPA9yRf1JgA9>HSx~TMB@?Pn?eBH!)M>joB6&s^WJ=$`#h(7RFr@V zzR~AX>zi$5Hs_+Aid^*-eZ)mwRI2!^UBxb8dn*aHTW0FKY9muPv-VqP3c4~>@T$aDYOS(aX9e31{31l-yP*S8;JeBQQTca8Ba8NLB`1+APex{9 zXHa4Qkw#1c*?nw`6H#yD@GbzA(Z0vu$LGWU$@93ziDvIQ zs+&e>tzI)Ny=Ix6ruM9wV+`-z@oce*=om+8=<5ax^LLqC0$)WZ-2z&R2rL{F;UE)} SH8fS8$PSYszL0$;;ly8iD&Eup diff --git a/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc b/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc index dd47d728e78f4e232280a8373cc399f66881cac5..ec01f5f764148eeed4674ec7494a783f6150a9be 100644 GIT binary patch delta 417 zcmYk1yGz4R6vlJ&PMTEO*kT`{ir@o5I%@~TRcooBaEF%ITcwz!l60v!b{6y^ihqF* z5TxMZpU_3uEKVJ|>D)o@DG1)-$N9eRo^v=4rRS14Fio9f&mU|YuV0!)%oD-oJpl`z zh(%Ar65Ty^jGNVou#u$kawTG}lG^R|VGse4U*%-m_@(3}$ zT%9A7>`(DKG9L)ErhcP1t!QhrG7AHRAreoT-7pYAn*bG%q>ox1CHlJl#UrafHY$jz z?f8u>ryx($(R6t+!^%k8a{NZqb(AXot-M{B8K)`qeGm<7x+M8a9? zG?=INsomJW{gh81v1P6F9jq2JuLueBEqj0x)XH5Wi;j&Uy~yt~J&2k`6Z{UG8wR4=ltjsVKK4!#$XH$GE(9KR5!24&%y- zp^QPatZ$&AVONT}Oant=q;=Z8gFrG#5nx(8Wi}xqQd8#;JEQMHWH+Sz$X@(@)NrLPj&ms&Px?FIE>|#O Ws3YPvH)qygNd1j02`xXPq5T6Sc2!{j diff --git a/api/webscraper/database.py b/api/webscraper/database.py index b322ada..209b557 100644 --- a/api/webscraper/database.py +++ b/api/webscraper/database.py @@ -1,27 +1,30 @@ import os +from dotenv import load_dotenv from datetime import datetime from dateutil import tz from supabase import create_client, Client from geocodio import GeocodioClient -from nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat -from nyiso_scraper import ( +from .nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat +from .nyiso_scraper import ( filter_nyiso_iq_sheet, filter_nyiso_cluster_sheet, filter_nyiso_in_service_sheet, filter_nyiso_withdrawn_sheets, ) -from ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted -from utils.scraper_utils import ( +from .ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted +from .utils.scraper_utils import ( geocode_lat_long, create_update_object, update_kdm, update_last_updated, ) -from database_constants import ( +from .database_constants import ( initial_kdm, ) +load_dotenv(os.path.join(os.path.dirname(__file__), "../../.env.local")) + url: str = os.environ.get("NEXT_PUBLIC_SUPABASE_URL") key: str = os.environ.get("NEXT_PUBLIC_SUPABASE_ANON_KEY") supabase: Client = create_client(url, key) @@ -48,7 +51,7 @@ def nyserda_large_to_database() -> None: In the case that the project is cancelled, we delete the project from the Supabase database. """ database = [] - database.extend(query_nyserda_large()) + database.extend(query_nyserda_large()[:10]) for project in database: if project.get("proposed_cod", None) is not None: ymd = datetime.strptime(project.get("proposed_cod"), "%Y").strftime( @@ -752,7 +755,7 @@ def ores_permitted_to_database() -> None: """ For testing """ -# nyserda_large_to_database() +nyserda_large_to_database() # nyserda_solar_to_database() # nyiso_to_database() # ores_noi_to_database() diff --git a/api/webscraper/nyiso_scraper.py b/api/webscraper/nyiso_scraper.py index 28192be..d6991ec 100644 --- a/api/webscraper/nyiso_scraper.py +++ b/api/webscraper/nyiso_scraper.py @@ -2,8 +2,8 @@ import pandas as pd from io import BytesIO import json -from utils.scraper_utils import clean_df_data -from database_constants import ( +from .utils.scraper_utils import clean_df_data +from .database_constants import ( renewable_energy_abbreviations, ) diff --git a/api/webscraper/nyserda_scraper.py b/api/webscraper/nyserda_scraper.py index 0ba7c3e..ae23966 100644 --- a/api/webscraper/nyserda_scraper.py +++ b/api/webscraper/nyserda_scraper.py @@ -1,7 +1,7 @@ import requests import json -from utils.scraper_utils import check_status, geocode_lat_long, standardize_label -from database_constants import renewable_energy_map, initial_kdm +from .utils.scraper_utils import check_status, geocode_lat_long, standardize_label +from .database_constants import renewable_energy_map, initial_kdm """ This scrapes data from the NYSERDA Large-scale Renewable Projects database. diff --git a/api/webscraper/ores_scraper.py b/api/webscraper/ores_scraper.py index 9daf72c..8c12bba 100644 --- a/api/webscraper/ores_scraper.py +++ b/api/webscraper/ores_scraper.py @@ -2,8 +2,8 @@ from bs4 import BeautifulSoup import pandas as pd from io import StringIO -from utils.scraper_utils import geocode_lat_long -from database_constants import initial_kdm +from .utils.scraper_utils import geocode_lat_long +from .database_constants import initial_kdm # url = "https://dps.ny.gov/ores-permit-applications" # page = requests.get(url) diff --git a/api/webscraper/server.py b/api/webscraper/server.py new file mode 100644 index 0000000..40ebe71 --- /dev/null +++ b/api/webscraper/server.py @@ -0,0 +1,50 @@ +from fastapi import FastAPI, Response +from database import * + +app = FastAPI() + + +@app.get("/favicon.ico") +def favicon(): + return Response(content=open("favicon.ico", "rb").read(), media_type="image/x-icon") + + +@app.get("/") +async def root(): + return {"message": "Hello World"} + + +@app.get("/nyserda_large") +async def run_nyserda_large(): + nyserda_large_to_database() + return {"message": "Hello NYSERDA Large"} + + +@app.get("/nyserda_solar") +async def run_nyserda_solar(): + nyserda_solar_to_database() + return {"message": "Hello NYSERDA Solar"} + + +@app.get("/nyiso") +async def run_nyiso(): + nyiso_to_database() + return {"message": "Hello NYISO"} + + +@app.get("/ores_noi") +async def run_ores_noi(): + ores_noi_to_database() + return {"message": "Hello ORES NOI"} + + +@app.get("/ores_under_review") +async def run_ores_under_review(): + ores_under_review_to_database() + return {"message": "Hello ORES Under Review"} + + +@app.get("/ores_permitted") +async def run_ores_permitted(): + ores_permitted_to_database() + return {"message": "Hello ORES Permitted"} diff --git a/next.config.mjs b/next.config.mjs index bfc08c1..abe7a9e 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -12,6 +12,14 @@ const nextConfig = { }, ], }, + async rewrites() { + return [ + { + source: '/api/:path*', + destination: 'http://127.0.0.1:5328/api/:path*', // Proxy to Backend + }, + ]; + }, }; export default nextConfig; From 696dba5075e7f558d5f7c152b2cf6da6124518fe Mon Sep 17 00:00:00 2001 From: Deena Sun Date: Fri, 22 Nov 2024 15:44:53 -0800 Subject: [PATCH 3/3] [feat] set up flask backend and api routes to trigger webscraper database actions --- api/app.py | 30 +++++++++++++++++++++++ api/webscraper/database.py | 2 +- api/webscraper/server.py | 50 -------------------------------------- 3 files changed, 31 insertions(+), 51 deletions(-) delete mode 100644 api/webscraper/server.py diff --git a/api/app.py b/api/app.py index bcbc87e..4de28d1 100644 --- a/api/app.py +++ b/api/app.py @@ -18,5 +18,35 @@ async def run_nyserda_large(): return {"message": "Hello NYSERDA Large"} +@app.route("/api/nyserda_solar", methods=["GET"]) +async def run_nyserda_solar(): + nyserda_solar_to_database() + return {"message": "Helo NYSERDA Solar"} + + +@app.route("/api/nyiso", methods=["GET"]) +async def run_nyiso(): + nyiso_to_database() + return {"message": "Hello NYISO"} + + +@app.route("/api/ores_noi", methods=["GET"]) +async def run_ores_noi(): + ores_noi_to_database() + return {"message": "Hello ORES NOI"} + + +@app.route("/api/ores_under_review", methods=["GET"]) +async def run_ores_under_review(): + ores_under_review_to_database() + return {"message": "Hello ORES Under Review"} + + +@app.route("/api/ores_permitted", methods=["GET"]) +async def run_ores_permitted(): + ores_permitted_to_database() + return {"message": "Hello ORES Permitted"} + + if __name__ == "__main__": app.run(port=5328, debug=True) diff --git a/api/webscraper/database.py b/api/webscraper/database.py index 209b557..24dad73 100644 --- a/api/webscraper/database.py +++ b/api/webscraper/database.py @@ -755,7 +755,7 @@ def ores_permitted_to_database() -> None: """ For testing """ -nyserda_large_to_database() +# nyserda_large_to_database() # nyserda_solar_to_database() # nyiso_to_database() # ores_noi_to_database() diff --git a/api/webscraper/server.py b/api/webscraper/server.py deleted file mode 100644 index 40ebe71..0000000 --- a/api/webscraper/server.py +++ /dev/null @@ -1,50 +0,0 @@ -from fastapi import FastAPI, Response -from database import * - -app = FastAPI() - - -@app.get("/favicon.ico") -def favicon(): - return Response(content=open("favicon.ico", "rb").read(), media_type="image/x-icon") - - -@app.get("/") -async def root(): - return {"message": "Hello World"} - - -@app.get("/nyserda_large") -async def run_nyserda_large(): - nyserda_large_to_database() - return {"message": "Hello NYSERDA Large"} - - -@app.get("/nyserda_solar") -async def run_nyserda_solar(): - nyserda_solar_to_database() - return {"message": "Hello NYSERDA Solar"} - - -@app.get("/nyiso") -async def run_nyiso(): - nyiso_to_database() - return {"message": "Hello NYISO"} - - -@app.get("/ores_noi") -async def run_ores_noi(): - ores_noi_to_database() - return {"message": "Hello ORES NOI"} - - -@app.get("/ores_under_review") -async def run_ores_under_review(): - ores_under_review_to_database() - return {"message": "Hello ORES Under Review"} - - -@app.get("/ores_permitted") -async def run_ores_permitted(): - ores_permitted_to_database() - return {"message": "Hello ORES Permitted"}