refactor(benefits): coalesce old and new property cols

deprecate old cols in mart definition
cal-itp · Sep 19, 2024 · 4c73df3 · 4c73df3
1 parent 855e6b4
commit 4c73df3
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 6 deletions.
diff --git a/warehouse/models/mart/benefits/_mart_benefits.yml b/warehouse/models/mart/benefits/_mart_benefits.yml
@@ -84,7 +84,13 @@ models:
       - name: event_properties_transit_agency
         description: The `transit_agency` value from the `event_properties` column
       - name: event_properties_eligibility_types
-        description: A semi-colon delimited list of `eligibility_types` values from the `event_properties` column
+        description: A semi-colon delimited list of `enrollment_flows` values from the `event_properties` column
+        deprecated:
+          enabled: true
+          reason: "This column is deprecated, use `event_properties_enrollment_flows` instead"
+          date: "2024-09-19"
+      - name: event_properties_enrollment_flows
+        description: A semi-colon delimited list of `enrollment_flows` values from the `event_properties` column
       - name: user_properties_eligibility_verifier
         description: The `eligibility_verifier` value from the `user_properties` column
       - name: user_properties_initial_referrer
@@ -100,4 +106,10 @@ models:
       - name: user_properties_user_agent
         description: The `user_agent` value from the `user_properties` column
       - name: user_properties_eligibility_types
-        description: A semi-colon delimited list of `eligibility_types` values from the `user_properties` column
+        description: A semi-colon delimited list of `enrollment_flows` values from the `user_properties` column
+        deprecated:
+          enabled: true
+          reason: "This column is deprecated, use `user_properties_enrollment_flows` instead"
+          date: "2024-09-19"
+      - name: user_properties_enrollment_flows
+        description: A semi-colon delimited list of `enrollment_flows` values from the `user_properties` column
diff --git a/warehouse/models/mart/benefits/fct_benefits_events.sql b/warehouse/models/mart/benefits/fct_benefits_events.sql
@@ -48,7 +48,15 @@ WITH fct_benefits_events AS (
         {{ json_extract_column('event_properties', 'payment_group') }},
         {{ json_extract_column('event_properties', 'status') }},
         {{ json_extract_column('event_properties', 'transit_agency') }},
-        {{ json_extract_flattened_column('event_properties', 'eligibility_types') }},
+
+        -- Historical data existed in `eligibility_types` but new data is in `enrollment_flows`
+        -- https://github.com/cal-itp/benefits/pull/2379
+        COALESCE(
+            {{ json_extract_flattened_column('event_properties', 'eligibility_types', no_alias = true) }},
+            {{ json_extract_flattened_column('event_properties', 'enrollment_flows', no_alias = true) }}
+        ) AS event_properties_enrollment_flows,
+        -- include the old field as well, deprecated in the mart definition
+        event_properties_enrollment_flows AS event_properties_eligibility_types,
 
         -- User Properties (https://app.amplitude.com/data/compiler/Benefits/properties/main/latest/user)
         {{ json_extract_column('user_properties', 'eligibility_verifier') }},
@@ -65,7 +73,15 @@ WITH fct_benefits_events AS (
         {{ json_extract_column('user_properties', 'referrer') }},
         {{ json_extract_column('user_properties', 'referring_domain') }},
         {{ json_extract_column('user_properties', 'user_agent') }},
-        {{ json_extract_flattened_column('user_properties', 'eligibility_types') }}
+
+        -- Historical data existed in `eligibility_types` but new data is in `enrollment_flows`
+        -- https://github.com/cal-itp/benefits/pull/2379
+        COALESCE(
+            {{ json_extract_flattened_column('user_properties', 'eligibility_types', no_alias = true) }},
+            {{ json_extract_flattened_column('user_properties', 'enrollment_flows', no_alias = true) }}
+        ) AS user_properties_enrollment_flows,
+        -- include the old field as well, deprecated in the mart definition
+        user_properties_enrollment_flows as user_properties_eligibility_types
 
     FROM {{ ref('stg_amplitude__benefits_events') }}
 ),
@@ -115,7 +131,8 @@ fct_old_enrollments AS (
     "5170d37b-43d5-4049-899c-b4d850e14990" as event_properties_payment_group,
     "success" as event_properties_status,
     "Monterey-Salinas Transit" as event_properties_transit_agency,
-    "senior" as event_properties_eligibility_types,
+    "senior" as event_properties_enrollment_flows,
+    event_properties_enrollment_flows as event_properties_eligibility_types,
     CASE
       WHEN client_event_time < '2022-08-12T07:00:00Z'
         THEN "DMV"
@@ -128,7 +145,8 @@ fct_old_enrollments AS (
     user_properties_user_agent,
     user_properties_referrer,
     user_properties_referring_domain,
-    "senior" as user_properties_eligibility_types
+    "senior" as user_properties_enrollment_flows,
+    user_properties_enrollment_flows as user_properties_eligibility_types
   FROM fct_benefits_events
   WHERE client_event_time >= '2021-12-08T08:00:00Z'
     and client_event_time < '2022-08-29T07:00:00Z'