Engagement/jessica/15896 radx mars deidentify (#16192)

Updating the radx mars deidentification transform. Updating the integration test and fixing an error
CDCgov · Oct 17, 2024 · a5a4b64 · a5a4b64
1 parent 1ff8ce4
commit a5a4b64
Show file tree

Hide file tree

Showing 3 changed files with 196 additions and 43 deletions.
diff --git a/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt b/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt
@@ -211,10 +211,7 @@ class ProcessFhirCommands : CliktCommand(
             }
 
             if (validationErrors.isNotEmpty()) {
-                return MessageOrBundle(
-                    filterErrors = mutableListOf(validationErrors.joinToString("\n")),
-                    filtersPassed = false
-                )
+                throw CliktError(validationErrors.joinToString("\n"))
             }
 
             receiver.conditionFilter.forEach { conditionFilter ->

diff --git a/prime-router/src/main/resources/metadata/hl7_mapping/ORU_R01/ORU_R01-deidentified.yml b/prime-router/src/main/resources/metadata/hl7_mapping/ORU_R01/ORU_R01-deidentified.yml
@@ -8,87 +8,135 @@ constants:
 elements:
   # PID 3.1
   - name: deidentified-patient-identifier
-    condition: '%patientPath.identifier[0].extension(%`rsext-cx-identifier`).extension.where(url = "CX.5").value != "PI"'
-    value: ['"<deidentified>"']
+    condition: '%patientPath.identifier[0].extension(%`rsext-cx-identifier`).extension.where(url = "CX.5").value !=
+     "PI" and %patientPath.identifier[0].extension(%`rsext-cx-identifier`).extension.where(url = "CX.5").value != 
+     "PT" and %patientPath.identifier[0].extension(%`rsext-cx-identifier`).extension.where(url = "CX.5").value != "SID"'
+    value: ['""']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-3-1]
 
   # PID 5.1
   - name: patient-last-name
     condition: '%patientPath.name[0].family.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-5-1]
 
   # PID 5.2
   - name: patient-first-name
     condition: '%patientPath.name[0].extension(%`rsext-xpn-human-name`).extension.where(url="XPN.2").value.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-5-2]
 
   # PID 5.3
   - name: patient-middle-name
     condition: '%patientPath.name[0].extension(%`rsext-xpn-human-name`).extension.where(url="XPN.3").value.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-5-3]
 
+  # PID 5.4
+  - name: patient-suffix
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/PATIENT/PID-5-4 ]
+
+  # PID 5.7
+  - name: patient-type-code
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/PATIENT/PID-5-7 ]
+
   #  # PID 7.1
   - name: deidentified-patient-dob
     condition: '%patientPath.birthDate.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-7-1]
 
   # PID 11.1
   - name: patient-address1
     condition: '%patientPath.address[0].extension(%`rsext-xad-address`).extension.where(url = %`rsext-sad-address-line`).extension.where(url = "SAD.1").value.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-11-1]
 
   # PID 11.2
   - name: patient-address2
     condition: '%patientPath.address[0].extension(%`rsext-xad-address`).extension.where(url = "XAD.2").value.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-11-2]
 
   # PID 11.3
   - name: patient-city
     condition: '%patientPath.address[0].city.empty().not()'
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-11-3]
 
-  # PID 11.5
-  - name: patient-postal-code
-    condition: '%patientPath.address[0].postalCode.empty().not()'
-    value: [ '"<deidentified>"' ]
-    hl7Spec: [ /PATIENT_RESULT/PATIENT/PID-11-5 ]
-
   # PID 13.4
   - name: patient-email
     condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`rsext-xtn-contact-point`).extension('XTN.4').value.empty().not()"
-    value: ['"<deidentified>"']
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-13-4]
 
   # PID 13.6
   - name: patient-phone-area-code
-    condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`ext-contactpoint-area`).value.empty().not()"
-    value: ['"<deidentified>"']
+    condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`ext-contactpoint-area`).value.empty().not() and %patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`ext-contactpoint-area`).value != '111'"
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-13-6]
 
   # PID 13.7
   - name: patient-local-phone
-    condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`rsext-xtn-contact-point`).extension('XTN.7').value.empty().not()"
-    value: ['"<deidentified>"']
+    condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`rsext-xtn-contact-point`).extension('XTN.7').value.empty().not() and %patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`rsext-xtn-contact-point`).extension('XTN.7').value != '1111111'"
+    value: ['"Deidentified"']
     hl7Spec: [/PATIENT_RESULT/PATIENT/PID-13-7]
 
-  # PID 13.12
-  - name: patient-unformatted-phone
-    condition: "%patientPath.telecom[0].where((use = 'home' or use = 'mobile')).extension(%`rsext-xtn-contact-point`).extension('XTN.12').value.empty().not()"
-    value: ['"<deidentified>"']
-    hl7Spec: [/PATIENT_RESULT/PATIENT/PID-13-12]
-
   # ORC 3.1
   - name: orc-filler-order-number
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/ORC-3-1]
 
+  # ORC 1
+  - name: order-control
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-1 ]
+
+  # ORC 2
+  - name: placer-order-number-orc-2
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-2 ]
+
+  # ORC 3.1
+  - name: filler-order-number-orc-3-1
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-3 ]
+
+  # ORC 3.2
+  - name: orc-3-2
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-3-2 ]
+
+  # ORC 3.3
+  - name: orc-3-3
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-3-3 ]
+
+  # ORC 3.4
+  - name: orc-3-4
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-3-4 ]
+
+  # ORC 4
+  - name: placer-group-number-orc-4
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-4 ]
+
+  # ORC 9
+  - name: datetime-of-order-event
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-9 ]
+
+  # ORC 12
+  - name: ordering-provider-from-practitioner
+    condition: 'false'
+
+  # ORC 12
+  - name: ordering-provider-from-practitioner-role
+    condition: 'false'
+
   # ORC 12.1
   - name: orc-ordering-provider-id
     value: ['""']
@@ -104,6 +152,16 @@ elements:
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/ORC-12-3]
 
+  # ORC 14
+  - name: call-back-phone-number-ocr-14
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-14 ]
+
+  # ORC 21
+  - name: ordering-facility-name-orc-21
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-21-1 ]
+
   # ORC 22.1
   - name: ordering-facility-street1
     value: ['""']
@@ -169,6 +227,14 @@ elements:
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/ORC-24-4]
 
+  # ORC 24
+  - name: ordering-provider-address-from-practitioner
+    condition: 'false'
+
+  # ORC 24
+  - name: ordering-provider-address-from-practitioner-role
+    condition: 'false'
+
   # ORC 24.5
   - name: ordering-provider-zip
     value: ['""']
@@ -179,16 +245,35 @@ elements:
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/ORC-24-9]
 
+  # ORC 25
+  - name: order-status-modifier-orc-25
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-25 ]
+
+  # ORC 29
+  - name: order-type-orc-29
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-29 ]
+
+  # ORC 33
+  - name: alternate-placer-order-number-orc-33
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/ORC-33 ]
+
+  # ORC ext
+  - name: orc-extension
+    condition: 'false'
+
+  # OBR 2.1
+  - name: placer-order-number-obr-2-1
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-2-1 ]
+
   # OBR 3.1
   - name: obr-filler-order-number
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/OBR-3-1]
 
-  # OBR 7.1
-  - name: observation-result-date
-    value: ['""']
-    hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/OBR-7-1]
-
   # OBR 16.1
   - name: obr-ordering-provider-id
     value: ['""']
@@ -204,10 +289,80 @@ elements:
     value: ['""']
     hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/OBR-16-3]
 
-  # OBR 22.1
-  - name: results-change-date
-    value: ['""']
-    hl7Spec: [/PATIENT_RESULT/ORDER_OBSERVATION/OBR-22-1]
+  # OBR 17.2
+  - name: obr-order-callback-telecom-use-code
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-17-2 ]
+
+  # OBR 17.3
+  - name: obr-order-callback-telecom-equipment-type
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-17-3 ]
+
+  # OBR 17.4
+  - name: obr-order-callback-email
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-17-4 ]
+
+  # OBR 17.6
+  - name: obr-order-callback-phone-area-code
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-17-6 ]
+
+  # OBR 17.7
+  - name: obr-order-callback-local-phone
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBR-17-7 ]
+
+  # OBX 14.1
+  - name: obx-observation-date-time
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-14-1 ]
+
+  # OBX 24.1
+  - name: obx-test-performing-org-street-addr
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-1 ]
+
+  # OBX 24.2
+  - name: obx-test-performing-org-street-addr-2
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-2 ]
+
+  # OBX 24.3
+  - name: obx-test-performing-org-city
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-3 ]
+
+  # OBX 24.4
+  - name: obx-test-performing-org-state
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-4 ]
+
+  # OBX 24.5
+  - name: obx-test-performing-org-zip-code
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-5 ]
+
+  # OBX 24.6
+  - name: obx-test-performing-org-country
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-6 ]
+
+  # OBX 24.7
+  - name: obx-test-performing-org-address-type
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-7 ]
+
+  # OBX 24.8
+  - name: obx-test-performing-org-or-geo-dest
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-8 ]
+
+  # OBX 24.9
+  - name: obx-test-performing-org-county
+    value: [ '""' ]
+    hl7Spec: [ /PATIENT_RESULT/ORDER_OBSERVATION/OBSERVATION/OBX-24-9 ]
 
   # Suppress all notes as they may contain PII data
   - name: patient-note
@@ -218,3 +373,5 @@ elements:
 
   - name: order-note
     condition: 'false'
+
+
diff --git a/.../src/testIntegration/resources/datatests/HL7_to_FHIR_to_HL7/ORU_deidentified-expected.hl7 b/.../src/testIntegration/resources/datatests/HL7_to_FHIR_to_HL7/ORU_deidentified-expected.hl7
@@ -1,9 +1,8 @@
 MSH|^~\&|MMTC.STAG^2.16.840.1.113883.3.8589.4.2.106.2^ISO|CAREEVOLUTION^00Z0000024^CLIA|AIMS.INTEGRATION.STG^2.16.840.1.114222.4.3.15.2^ISO|AIMS.PLATFORM^2.16.840.1.114222.4.1.217446^ISO|20240412110603-0500||ORU^R01^ORU_R01|20240412110603_ff98cc992d5146e7916a5f0b873e534f|T|2.5.1|||NE|NE|||||PHLabReport-NoAck^ELR251R1_Rcvr_Prof^2.16.840.1.113883.9.11^ISO
 SFT|CAREEVOLUTION|2022|MMTC.STAG|16948||20240311
-PID|1||92041f50874c4595955d47f7ae4981c5^^^MMTC.STAG&2.16.840.1.113883.3.8589.4.2.106.2&ISO^PI||<deidentified>^<deidentified>^<deidentified>^^^^S||<deidentified>||||<deidentified>^<deidentified>^<deidentified>^PIDSTATE!^<deidentified>^USA^^^PIDCOUNTY!||^^^<deidentified>^^<deidentified>^<deidentified>^^^^^<deidentified>
-ORC|RE||^MMTC.STAG^2.16.840.1.113883.3.8589.4.2.106.2^ISO||||||||||||||||||SA.OTCSelfReport
-OBR|1||^MMTC.STAG^2.16.840.1.113883.3.8589.4.2.106.2^ISO|97097-0^SARS-CoV-2 (COVID-19) Ag [Presence] in Upper respiratory specimen by Rapid immunoassay^LN^^^^2.71|||||||||||||||||||||F
-OBX|1|CWE|97097-0^SARS-CoV-2 (COVID-19) Ag [Presence] in Upper respiratory specimen by Rapid immunoassay^LN^^^^2.71||260373001^Detected^SCT^^^^20200901||||||F||||00Z0000042||BD Veritor At-Home COVID-19 Test_Becton, Dickinson and Company (BD)_EUA^^99ELR^^^^Vunknown||20240412230603||||SA.OTCSelfReport^^^^^&2.16.840.1.113883.3.8589.4.1.152&ISO^XX^^^00Z0000042|14 Fake AtHome Test Street^^Fake City
+PID|1||92041f50874c4595955d47f7ae4981c5^^^MMTC.STAG&2.16.840.1.113883.3.8589.4.2.106.2&ISO^PI||Deidentified^Deidentified^Deidentified||Deidentified||||Deidentified^Deidentified^Deidentified^PIDSTATE!^PIDPOSTAL!^USA^^^PIDCOUNTY!||^^^Deidentified^^Deidentified^Deidentified^^^^^PIDUNFORMATPHONE!
+OBR|1||^MMTC.STAG^2.16.840.1.113883.3.8589.4.2.106.2^ISO|97097-0^SARS-CoV-2 (COVID-19) Ag [Presence] in Upper respiratory specimen by Rapid immunoassay^LN^^^^2.71|||DATE!|||||||||||||||DATE!|||F
+OBX|1|CWE|97097-0^SARS-CoV-2 (COVID-19) Ag [Presence] in Upper respiratory specimen by Rapid immunoassay^LN^^^^2.71||260373001^Detected^SCT^^^^20200901||||||F||||00Z0000042||BD Veritor At-Home COVID-19 Test_Becton, Dickinson and Company (BD)_EUA^^99ELR^^^^Vunknown||20240412230603||||SA.OTCSelfReport^^^^^&2.16.840.1.113883.3.8589.4.1.152&ISO^XX^^^00Z0000042
 OBX|2|NM|35659-2^Age at specimen collection^LN^^^^2.71||44|a^year^UCUM^^^^2.1|||||F||||00Z0000042||||||||SA.OTCSelfReport^^^^^&2.16.840.1.113883.3.8589.4.1.152&ISO^XX^^^00Z0000042|14 Fake AtHome Test Street^^Fake City|||||QST
 OBX|3|CWE|95419-8^Whether patient has symptoms related to condition of interest^LN^^^^2.71||Y^Yes^HL70136^^^^2.5.1||||||F||||00Z0000042||||||||SA.OTCSelfReport^^^^^&2.16.840.1.113883.3.8589.4.1.152&ISO^XX^^^00Z0000042|14 Fake AtHome Test Street^^Fake City|||||QST
 OBX|4|DT|65222-2^Date and time of symptom onset^LN^^^^2.71||20240411||||||F||||00Z0000042||||||||SA.OTCSelfReport^^^^^&2.16.840.1.113883.3.8589.4.1.152&ISO^XX^^^00Z0000042|14 Fake AtHome Test Street^^Fake City|||||QST