checking before refactor

Author: charlie-costanzo

airflow/dags/scrape_state_geoportal/state_highway_network.yml

@@ -4,7 +4,4 @@ root_url: 'https://caltrans-gis.dot.ca.gov/arcgis/rest/services/'
 service: "CHhighway/SHN_Lines"
 layer: "0"
 product: 'state_highway_network'
-where: "1=1"  # You can change this to filter data
-outFields: "*"  # Specify the fields to return
-f: "geojson"  # Format of the response
 resultRecordCount: 2000

airflow/plugins/operators/scrape_state_geoportal.py

@@ -1,6 +1,7 @@
 import gzip
 import logging
-import os
+
+# import os
 from typing import ClassVar, List
 
 import pandas as pd  # type: ignore
@@ -11,7 +12,8 @@
 
 from airflow.models import BaseOperator  # type: ignore
 
-API_BUCKET = os.environ["CALITP_BUCKET__STATE_GEOPORTAL_DATA_PRODUCTS"]
+API_BUCKET = "gs://calitp-state-geoportal-scrape"
+# API_BUCKET = os.environ["CALITP_BUCKET__STATE_GEOPORTAL_DATA_PRODUCTS"]
 
 
 class StateGeoportalAPIExtract(PartitionedGCSArtifact):
@@ -86,7 +88,7 @@ def fetch_from_state_geoportal(self):
                 params["resultOffset"] = offset
 
                 # Make the request
-                response = requests.get(validated_url, params=params)
+                response = requests.get(validated_url, params=params).raise_for_status()
                 data = response.json()
 
                 # Break the loop if there are no more features
@@ -145,8 +147,6 @@ class StateGeoportalAPIOperator(BaseOperator):
         "root_url",
         "service",
         "layer",
-        "where",
-        "outFields",
         "resultRecordCount",
     )
 
@@ -156,17 +156,13 @@ def __init__(
         root_url,
         service,
         layer,
-        where,
-        outFields,
         resultRecordCount,
         **kwargs,
     ):
         self.product = product
         self.root_url = root_url
         self.service = service
         self.layer = layer
-        self.where = where
-        self.outFields = outFields
         self.resultRecordCount = resultRecordCount
 
         """An operator that extracts and saves JSON data from the State Geoportal
@@ -178,8 +174,6 @@ def __init__(
             root_url=self.root_url,
             service=self.service,
             product=f"{self.product}_geodata",
-            where=self.where,
-            outFields=self.outFields,
             layer=self.layer,
             resultRecordCount=self.resultRecordCount,
             filename=f"{self.product}_geodata.jsonl.gz",
@@ -193,7 +187,7 @@ def execute(self, **kwargs):
         df = pd.json_normalize(api_content)
 
         if self.product == "state_highway_network":
-            # Select columns to keep
+            # Select columns to keep, have to be explicit because there are duplicate values after normalizing
             df = df[
                 [
                     "properties.Route",