Move MedReq and Allergy references to end of table (#295)

CONTRIBUTING.md

@@ -31,6 +31,14 @@ Things to keep in mind:
 - If the new resource links to Encounter,
   add it to the completion checking done in the Encounter code.
   (Though consider the effect this will have on existing encounters.)
+- Try to keep the columns in spec-order,
+  except please move any isolated reference fields to the end,
+  since they are not really human-readable and
+  this makes it nicer to visually scan the table.
+- Add a self-Reference field for easier joining.
+  For example, if you're adding a `condition` table, in addition to
+  the `id` field, add a `condition_ref` field defined like:
+  `concat('Condition/', id) AS condition_ref`
 
 ## Rebuilding the reference SQL
 

cumulus_library/studies/core/core_templates/allergyintolerance.sql.jinja

@@ -56,7 +56,7 @@ temp_category AS (
         t.category
     FROM
         allergyintolerance AS a,
-        UNNEST(a.category) AS t (category)
+        unnest(a.category) AS t (category)
 ),
 
 flattened_reaction AS ({{ unnest_utils.flatten('allergyintolerance', 'reaction') }}),
@@ -90,7 +90,6 @@ temp_reaction AS (
 
 SELECT
     ta.id,
-    CONCAT('AllergyIntolerance/', ta.id) AS allergyintolerance_ref,
 
     {#- We don't expose system for these next two since we filter
         down to a single system in the denormalization table #}
@@ -107,9 +106,6 @@ SELECT
     dn_code.system AS code_system,
     dn_code.display AS code_display,
 
-    ta.patient_ref,
-    ta.encounter_ref,
-
     {#- recordedDate is not in US Core.
         But it's useful for looking at only a study period of data. #}
     ta.recordedDate,
@@ -126,7 +122,11 @@ SELECT
     tr.manifestation_code AS reaction_manifestation_code,
     tr.manifestation_system AS reaction_manifestation_system,
     tr.manifestation_display AS reaction_manifestation_display,
-    tr.severity AS reaction_severity
+    tr.severity AS reaction_severity,
+
+    concat('AllergyIntolerance/', ta.id) AS allergyintolerance_ref,
+    ta.patient_ref,
+    ta.encounter_ref
 
 FROM temp_allergyintolerance AS ta
 LEFT JOIN temp_reaction AS tr ON ta.id = tr.id
@@ -135,4 +135,4 @@ LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_
 LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
     ON ta.id = dn_vstat.id
 LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
-WHERE ta.recordedDate BETWEEN DATE('2016-01-01') AND CURRENT_DATE;
+WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;

cumulus_library/studies/core/core_templates/medicationrequest.sql.jinja

@@ -64,60 +64,69 @@ CREATE TABLE core__medicationrequest AS (
         WHERE mr.med_ref IS NOT NULL AND {{ syntax.like('mr.med_ref', 'Medication/%') }}
     ),
 
-    {# this holds all the fields that each unioned table below will share -#}
-    mr_shared AS (
+    unified_codes AS (
+        {# Internal: medication data from inline ETL extraction.
+        95% of the time, this dataset is the 'correct' dataset for Cerner.
+        It may not be present in EPIC datasets. #}
         SELECT
             mr.id,
-            mr.status,
-            mr.intent,
-            mrc.code AS category_code,
-            mrc.system AS category_system,
-            mrc.display AS category_display,
-            mr.reportedBoolean,
-            mr.reported_ref,
-            mr.subject_ref,
-            mr.encounter_ref,
-            mr.authoredOn,
-            mr.authoredOn_month
+            mric.code AS medication_code,
+            mric.system AS medication_system,
+            mric.display AS medication_display
         FROM mr_basics AS mr
-        LEFT JOIN core__medicationrequest_dn_category AS mrc ON mr.id = mrc.id
+        INNER JOIN core__medicationrequest_dn_inline_code AS mric ON mr.id = mric.id
+
+        {# Contained: medication reference into contained resources.
+        We've also seen this in Cerner. Though in the cases we've seen this, it's usually
+        a very lightly fleshed out resource, with only code.text and no codings. #}
+        UNION
+        SELECT
+            mr.id,
+            mrcc.code AS medication_code,
+            mrcc.system AS medication_system,
+            mrcc.display AS medication_display
+        FROM mr_basics AS mr
+        INNER JOIN contained_refs AS cr ON mr.id = cr.id
+        INNER JOIN core__medicationrequest_dn_contained_code AS mrcc
+            ON cr.id = mrcc.id AND cr.medication_id = mrcc.contained_id
+        WHERE mrcc.resource_type = 'Medication'
+
+        {# External: medication by reference from external medications table.
+        This is generally how we expect EPIC to provide medication data. #}
+        UNION
+        SELECT
+            mr.id,
+            mc.code AS medication_code,
+            mc.system AS medication_system,
+            mc.display AS medication_display
+        FROM mr_basics AS mr
+        INNER JOIN external_refs AS er ON mr.id = er.id
+        INNER JOIN core__medication_dn_code AS mc ON er.medication_id = mc.id
     )
 
-    {# Internal: medication data from inline ETL extraction.
-    95% of the time, this dataset is the 'correct' dataset for Cerner.
-    It may not be present in EPIC datasets. #}
     SELECT
-        mr.*,
-        mric.code AS medication_code,
-        mric.system AS medication_system,
-        mric.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN core__medicationrequest_dn_inline_code AS mric ON mr.id = mric.id
+        mr.id,
+        mr.status,
+        mr.intent,
 
-    {# Contained: medication reference into contained resources.
-    We've also seen this in Cerner. Though in the cases we've seen this, it's usually
-    a very lightly fleshed out resource, with only code.text and no codings. #}
-    UNION
-    SELECT
-        mr.*,
-        mrcc.code AS medication_code,
-        mrcc.system AS medication_system,
-        mrcc.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN contained_refs AS cr ON mr.id = cr.id
-    INNER JOIN core__medicationrequest_dn_contained_code AS mrcc
-        ON cr.id = mrcc.id AND cr.medication_id = mrcc.contained_id
-    WHERE mrcc.resource_type = 'Medication'
+        mrc.code AS category_code,
+        mrc.system AS category_system,
+        mrc.display AS category_display,
 
-    {# External: medication by reference from external medications table.
-    This is generally how we expect EPIC to provide medication data. #}
-    UNION
-    SELECT
-        mr.*,
-        mc.code AS medication_code,
-        mc.system AS medication_system,
-        mc.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN external_refs AS er ON mr.id = er.id
-    INNER JOIN core__medication_dn_code AS mc ON er.medication_id = mc.id
+        mr.reportedBoolean,
+        mr.reported_ref,
+
+        uc.medication_code,
+        uc.medication_system,
+        uc.medication_display,
+
+        mr.authoredOn,
+        mr.authoredOn_month,
+
+        concat('MedicationRequest/', mr.id) AS medicationrequest_ref,
+        mr.subject_ref,
+        mr.encounter_ref
+    FROM mr_basics AS mr
+    LEFT JOIN unified_codes AS uc ON mr.id = uc.id
+    LEFT JOIN core__medicationrequest_dn_category AS mrc ON mr.id = mrc.id
 );

cumulus_library/studies/core/reference_sql/builder_allergyintolerance.sql

@@ -273,7 +273,7 @@ temp_category AS (
         t.category
     FROM
         allergyintolerance AS a,
-        UNNEST(a.category) AS t (category)
+        unnest(a.category) AS t (category)
 ),
 
 flattened_reaction AS (SELECT DISTINCT
@@ -304,7 +304,6 @@ temp_reaction AS (
 
 SELECT
     ta.id,
-    CONCAT('AllergyIntolerance/', ta.id) AS allergyintolerance_ref,
     dn_cstat.code AS clinicalStatus_code,
     dn_vstat.code AS verificationStatus_code,
     ta.type,
@@ -314,9 +313,6 @@ SELECT
     dn_code.code AS code_code,
     dn_code.system AS code_system,
     dn_code.display AS code_display,
-
-    ta.patient_ref,
-    ta.encounter_ref,
     ta.recordedDate,
     ta.recordedDate_week,
     ta.recordedDate_month,
@@ -328,7 +324,11 @@ SELECT
     tr.manifestation_code AS reaction_manifestation_code,
     tr.manifestation_system AS reaction_manifestation_system,
     tr.manifestation_display AS reaction_manifestation_display,
-    tr.severity AS reaction_severity
+    tr.severity AS reaction_severity,
+
+    concat('AllergyIntolerance/', ta.id) AS allergyintolerance_ref,
+    ta.patient_ref,
+    ta.encounter_ref
 
 FROM temp_allergyintolerance AS ta
 LEFT JOIN temp_reaction AS tr ON ta.id = tr.id
@@ -337,4 +337,4 @@ LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_
 LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
     ON ta.id = dn_vstat.id
 LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
-WHERE ta.recordedDate BETWEEN DATE('2016-01-01') AND CURRENT_DATE;
+WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;

cumulus_library/studies/core/reference_sql/builder_encounter.sql

@@ -582,7 +582,8 @@ temp_encounter_completion AS (
         SELECT
             ece.encounter_id,
             (
-                BOOL_OR(ec.table_name = 'condition')
+                BOOL_OR(ec.table_name = 'allergyintolerance')
+                AND BOOL_OR(ec.table_name = 'condition')
                 AND BOOL_OR(ec.table_name = 'documentreference')
                 AND BOOL_OR(ec.table_name = 'medicationrequest')
                 AND BOOL_OR(ec.table_name = 'observation')
@@ -734,7 +735,8 @@ temp_encounter_completion AS (
         SELECT
             ece.encounter_id,
             (
-                BOOL_OR(ec.table_name = 'condition')
+                BOOL_OR(ec.table_name = 'allergyintolerance')
+                AND BOOL_OR(ec.table_name = 'condition')
                 AND BOOL_OR(ec.table_name = 'documentreference')
                 AND BOOL_OR(ec.table_name = 'medicationrequest')
                 AND BOOL_OR(ec.table_name = 'observation')

cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql

@@ -154,54 +154,64 @@ CREATE TABLE core__medicationrequest AS (
         WHERE mr.med_ref IS NOT NULL AND REGEXP_LIKE(mr.med_ref, '^Medication/.*$')
     ),
 
-    mr_shared AS (
+    unified_codes AS (
+
+        SELECT
+            mr.id,
+            mric.code AS medication_code,
+            mric.system AS medication_system,
+            mric.display AS medication_display
+        FROM mr_basics AS mr
+        INNER JOIN core__medicationrequest_dn_inline_code AS mric ON mr.id = mric.id
+
+
+        UNION
+        SELECT
+            mr.id,
+            mrcc.code AS medication_code,
+            mrcc.system AS medication_system,
+            mrcc.display AS medication_display
+        FROM mr_basics AS mr
+        INNER JOIN contained_refs AS cr ON mr.id = cr.id
+        INNER JOIN core__medicationrequest_dn_contained_code AS mrcc
+            ON cr.id = mrcc.id AND cr.medication_id = mrcc.contained_id
+        WHERE mrcc.resource_type = 'Medication'
+
+
+        UNION
         SELECT
             mr.id,
-            mr.status,
-            mr.intent,
-            mrc.code AS category_code,
-            mrc.system AS category_system,
-            mrc.display AS category_display,
-            mr.reportedBoolean,
-            mr.reported_ref,
-            mr.subject_ref,
-            mr.encounter_ref,
-            mr.authoredOn,
-            mr.authoredOn_month
+            mc.code AS medication_code,
+            mc.system AS medication_system,
+            mc.display AS medication_display
         FROM mr_basics AS mr
-        LEFT JOIN core__medicationrequest_dn_category AS mrc ON mr.id = mrc.id
+        INNER JOIN external_refs AS er ON mr.id = er.id
+        INNER JOIN core__medication_dn_code AS mc ON er.medication_id = mc.id
     )
 
-
     SELECT
-        mr.*,
-        mric.code AS medication_code,
-        mric.system AS medication_system,
-        mric.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN core__medicationrequest_dn_inline_code AS mric ON mr.id = mric.id
-
-
-    UNION
-    SELECT
-        mr.*,
-        mrcc.code AS medication_code,
-        mrcc.system AS medication_system,
-        mrcc.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN contained_refs AS cr ON mr.id = cr.id
-    INNER JOIN core__medicationrequest_dn_contained_code AS mrcc
-        ON cr.id = mrcc.id AND cr.medication_id = mrcc.contained_id
-    WHERE mrcc.resource_type = 'Medication'
-
-
-    UNION
-    SELECT
-        mr.*,
-        mc.code AS medication_code,
-        mc.system AS medication_system,
-        mc.display AS medication_display
-    FROM mr_shared AS mr
-    INNER JOIN external_refs AS er ON mr.id = er.id
-    INNER JOIN core__medication_dn_code AS mc ON er.medication_id = mc.id
+        mr.id,
+        mr.status,
+        mr.intent,
+
+        mrc.code AS category_code,
+        mrc.system AS category_system,
+        mrc.display AS category_display,
+
+        mr.reportedBoolean,
+        mr.reported_ref,
+
+        uc.medication_code,
+        uc.medication_system,
+        uc.medication_display,
+
+        mr.authoredOn,
+        mr.authoredOn_month,
+
+        concat('MedicationRequest/', mr.id) AS medicationrequest_ref,
+        mr.subject_ref,
+        mr.encounter_ref
+    FROM mr_basics AS mr
+    LEFT JOIN unified_codes AS uc ON mr.id = uc.id
+    LEFT JOIN core__medicationrequest_dn_category AS mrc ON mr.id = mrc.id
 );