Deenasun/81 feat schedule the web scraper

api/app.py

@@ -0,0 +1,52 @@
+from flask import Flask
+from webscraper.database import *
+from webscraper.nyserda_scraper import *
+
+app = Flask(__name__)
+
+# flask run --port 5328 --debug
+
+
+@app.route("/api/hello", methods=["GET"])
+def hello_world():
+    return "Hello, World!"
+
+
+@app.route("/api/nyserda_large", methods=["GET"])
+async def run_nyserda_large():
+    nyserda_large_to_database()
+    return {"message": "Hello NYSERDA Large"}
+
+
+@app.route("/api/nyserda_solar", methods=["GET"])
+async def run_nyserda_solar():
+    nyserda_solar_to_database()
+    return {"message": "Helo NYSERDA Solar"}
+
+
+@app.route("/api/nyiso", methods=["GET"])
+async def run_nyiso():
+    nyiso_to_database()
+    return {"message": "Hello NYISO"}
+
+
+@app.route("/api/ores_noi", methods=["GET"])
+async def run_ores_noi():
+    ores_noi_to_database()
+    return {"message": "Hello ORES NOI"}
+
+
+@app.route("/api/ores_under_review", methods=["GET"])
+async def run_ores_under_review():
+    ores_under_review_to_database()
+    return {"message": "Hello ORES Under Review"}
+
+
+@app.route("/api/ores_permitted", methods=["GET"])
+async def run_ores_permitted():
+    ores_permitted_to_database()
+    return {"message": "Hello ORES Permitted"}
+
+
+if __name__ == "__main__":
+    app.run(port=5328, debug=True)

api/webscraper/database.py

@@ -1,27 +1,30 @@
 import os
+from dotenv import load_dotenv
 from datetime import datetime
 from dateutil import tz
 from supabase import create_client, Client
 from geocodio import GeocodioClient
 
-from nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat
-from nyiso_scraper import (
+from .nyserda_scraper import query_nyserda_large, query_nyserda_solar_repeat
+from .nyiso_scraper import (
     filter_nyiso_iq_sheet,
     filter_nyiso_cluster_sheet,
     filter_nyiso_in_service_sheet,
     filter_nyiso_withdrawn_sheets,
 )
-from ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted
-from utils.scraper_utils import (
+from .ores_scraper import query_ores_noi, query_ores_under_review, query_ores_permitted
+from .utils.scraper_utils import (
     geocode_lat_long,
     create_update_object,
     update_kdm,
     update_last_updated,
 )
-from database_constants import (
+from .database_constants import (
     initial_kdm,
 )
 
+load_dotenv(os.path.join(os.path.dirname(__file__), "../../.env.local"))
+
 url: str = os.environ.get("NEXT_PUBLIC_SUPABASE_URL")
 key: str = os.environ.get("NEXT_PUBLIC_SUPABASE_ANON_KEY")
 supabase: Client = create_client(url, key)
@@ -48,7 +51,7 @@ def nyserda_large_to_database() -> None:
     In the case that the project is cancelled, we delete the project from the Supabase database.
     """
     database = []
-    database.extend(query_nyserda_large())
+    database.extend(query_nyserda_large()[:10])
     for project in database:
         if project.get("proposed_cod", None) is not None:
             ymd = datetime.strptime(project.get("proposed_cod"), "%Y").strftime(

api/webscraper/nyiso_scraper.py

@@ -2,8 +2,8 @@
 import pandas as pd
 from io import BytesIO
 import json
-from utils.scraper_utils import clean_df_data
-from database_constants import (
+from .utils.scraper_utils import clean_df_data
+from .database_constants import (
     renewable_energy_abbreviations,
 )
 

api/webscraper/nyserda_scraper.py

@@ -1,7 +1,7 @@
 import requests
 import json
-from utils.scraper_utils import check_status, geocode_lat_long, standardize_label
-from database_constants import renewable_energy_map, initial_kdm
+from .utils.scraper_utils import check_status, geocode_lat_long, standardize_label
+from .database_constants import renewable_energy_map, initial_kdm
 
 """
 This scrapes data from the NYSERDA Large-scale Renewable Projects database.

api/webscraper/ores_scraper.py

@@ -2,8 +2,8 @@
 from bs4 import BeautifulSoup
 import pandas as pd
 from io import StringIO
-from utils.scraper_utils import geocode_lat_long
-from database_constants import initial_kdm
+from .utils.scraper_utils import geocode_lat_long
+from .database_constants import initial_kdm
 
 # url = "https://dps.ny.gov/ores-permit-applications"
 # page = requests.get(url)

next.config.mjs

@@ -12,6 +12,14 @@ const nextConfig = {
       },
     ],
   },
+  async rewrites() {
+    return [
+      {
+        source: '/api/:path*',
+        destination: 'http://127.0.0.1:5328/api/:path*', // Proxy to Backend
+      },
+    ];
+  },
 };
 
 export default nextConfig;