From 6eece1f214616afdfc374e61c5f42c22895ae12f Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Thu, 8 Feb 2024 11:51:56 -0500 Subject: [PATCH 1/5] Expanded code system preference lists --- cumulus_library/studies/core/builder_condition.py | 3 +++ cumulus_library/studies/core/builder_encounter.py | 11 +++++++++++ cumulus_library/studies/discovery/code_detection.py | 2 +- .../codeable_concept_denormalize.sql.jinja | 2 +- cumulus_library/template_sql/sql_utils.py | 7 ++++--- tests/test_templates.py | 4 ++-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/cumulus_library/studies/core/builder_condition.py b/cumulus_library/studies/core/builder_condition.py index 1492f509..d085818c 100644 --- a/cumulus_library/studies/core/builder_condition.py +++ b/cumulus_library/studies/core/builder_condition.py @@ -54,6 +54,9 @@ def denormalize_codes(self): "http://snomed.info/sct", "http://hl7.org/fhir/sid/icd-10-cm", "http://hl7.org/fhir/sid/icd-9-cm", + "urn:oid:1.2.840.114350.1.13.71.2.7.2", + "urn:oid:1.2.840.114350.1.13.71.2.7.4", + "http://terminology.hl7.org/CodeSystem/data-absent-reason", ], ) self.queries.append( diff --git a/cumulus_library/studies/core/builder_encounter.py b/cumulus_library/studies/core/builder_encounter.py index cd3552cd..154c5df6 100644 --- a/cumulus_library/studies/core/builder_encounter.py +++ b/cumulus_library/studies/core/builder_encounter.py @@ -40,6 +40,9 @@ def denormalize_codes(self, schema, cursor): "http://terminology.hl7.org/CodeSystem/v2-0004", "urn:oid:2.16.840.1.113883.4.642.3.248", "http://snomed.info/sct", + "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/71", + "urn:oid:1.2.840.114350.1.13.71.2.7.10", + "urn:oid:1.2.840.114350.1.13.71.2.7.2", ], "has_data": False, }, @@ -51,6 +54,8 @@ def denormalize_codes(self, schema, cursor): "http://terminology.hl7.org/CodeSystem/service-type", "urn:oid:2.16.840.1.113883.4.642.3.518", "http://snomed.info/sct", + "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/34", + "urn:oid:1.2.840.114350.1.13.71.2.7.10", ], "has_data": False, }, @@ -61,6 +66,8 @@ def denormalize_codes(self, schema, cursor): "code_systems": [ "http://terminology.hl7.org/CodeSystem/v3-ActPriority", "http://snomed.info/sct", + "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/3", + "urn:oid:1.2.840.114350.1.13.71.2.7.10", ], "has_data": False, }, @@ -71,6 +78,10 @@ def denormalize_codes(self, schema, cursor): "code_systems": [ "http://terminology.hl7.org/CodeSystem/v3-ActPriority", "http://snomed.info/sct", + "http://hl7.org/fhir/sid/icd-10-cm", + "http://hl7.org/fhir/sid/icd-9-cm", + "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/nomenclature", + "urn:oid:1.2.840.114350.1.13.71.2.7.2", ], "has_data": False, }, diff --git a/cumulus_library/studies/discovery/code_detection.py b/cumulus_library/studies/discovery/code_detection.py index f73d96d7..5642c97d 100644 --- a/cumulus_library/studies/discovery/code_detection.py +++ b/cumulus_library/studies/discovery/code_detection.py @@ -8,7 +8,7 @@ class CodeDetectionBuilder(base_table_builder.BaseTableBuilder): display_text = "Selecting unique code systems..." - def _check_coding_against_db(code_source, schema, cursor): + def _check_coding_against_db(self, code_source, schema, cursor): """selects the appropriate DB query to run""" if code_source["is_array"]: diff --git a/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja b/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja index e954cd57..37a1c60f 100644 --- a/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja +++ b/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja @@ -21,7 +21,7 @@ CREATE TABLE {{ target_table }} AS ( {%- endif %} {%- if filter_priority %} WHERE - u.codeable_concept.system = '{{ system }}' + u.codeable_concept.system LIKE '{{ system }}%' {%- endif %} ), --noqa: LT07 {%- endfor %} diff --git a/cumulus_library/template_sql/sql_utils.py b/cumulus_library/template_sql/sql_utils.py index 797bc330..e07c00b6 100644 --- a/cumulus_library/template_sql/sql_utils.py +++ b/cumulus_library/template_sql/sql_utils.py @@ -28,8 +28,9 @@ class CodeableConceptConfig: :param source_id: the id field to use in the new table (default: 'id') :param filter_priority: If true, will use code systems to select a single code, in preference order, for use as a display value. - :param code_systems: a list of systems, in preference order, for selecting data - for filtering. This should not be set if filter_priority is false. + :param code_systems: a list of strings matching the start of the systems field, + in preference order, for selecting data for filtering. This should not be set + if filter_priority is false. """ column_name: str @@ -50,7 +51,7 @@ class ExtensionConfig: :param target_table: the name of the table to create :param target_col_prefix: the string to prepend code/display column names with :param fhir_extension: the URL of the FHIR resource to select - :param code_systems: a list of codes, in preference order, to use to select data + :param ext_systems: a list of codes, in preference order, to use to select data :param is_array: a boolean indicating if the targeted field is an array type """ diff --git a/tests/test_templates.py b/tests/test_templates.py index ab89b3ec..d4ad4877 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -77,7 +77,7 @@ def test_codeable_concept_denormalize_filter_creation(): source AS s, UNNEST(s.code_col.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://snomed.info/sct' + u.codeable_concept.system LIKE 'http://snomed.info/sct%' ), --noqa: LT07 system_code_col_1 AS ( @@ -91,7 +91,7 @@ def test_codeable_concept_denormalize_filter_creation(): source AS s, UNNEST(s.code_col.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://hl7.org/fhir/sid/icd-10-cm' + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-10-cm%' ), --noqa: LT07 union_table AS ( From 53c793088547f46d01a060d2ae825a9e71aa16be Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Thu, 8 Feb 2024 14:16:24 -0500 Subject: [PATCH 2/5] Updated for stricter system matching --- .../studies/core/builder_condition.py | 7 +++-- .../studies/core/builder_encounter.py | 30 ++++++++++++------- .../codeable_concept_denormalize.sql.jinja | 2 +- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/cumulus_library/studies/core/builder_condition.py b/cumulus_library/studies/core/builder_condition.py index d085818c..0ffa52cf 100644 --- a/cumulus_library/studies/core/builder_condition.py +++ b/cumulus_library/studies/core/builder_condition.py @@ -54,9 +54,12 @@ def denormalize_codes(self): "http://snomed.info/sct", "http://hl7.org/fhir/sid/icd-10-cm", "http://hl7.org/fhir/sid/icd-9-cm", - "urn:oid:1.2.840.114350.1.13.71.2.7.2", - "urn:oid:1.2.840.114350.1.13.71.2.7.4", + "http://hl7.org/fhir/sid/icd-9-cm/diagnosis", + # Cerner specific systems "http://terminology.hl7.org/CodeSystem/data-absent-reason", + # EPIC specific systems + "urn:oid:1.2.840.114350.1.13.71.2.7.2.728286", + "urn:oid:1.2.840.114350.1.13.71.2.7.4.698084.10375", ], ) self.queries.append( diff --git a/cumulus_library/studies/core/builder_encounter.py b/cumulus_library/studies/core/builder_encounter.py index 154c5df6..ebcf68ab 100644 --- a/cumulus_library/studies/core/builder_encounter.py +++ b/cumulus_library/studies/core/builder_encounter.py @@ -40,9 +40,13 @@ def denormalize_codes(self, schema, cursor): "http://terminology.hl7.org/CodeSystem/v2-0004", "urn:oid:2.16.840.1.113883.4.642.3.248", "http://snomed.info/sct", - "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/71", - "urn:oid:1.2.840.114350.1.13.71.2.7.10", - "urn:oid:1.2.840.114350.1.13.71.2.7.2", + # Cerner specific systems + "https://fhir.cerner.com/%/codeSet/71", + # EPIC specific systems + "urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.10110", + "urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.18875", + "urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.30", + "urn:oid:1.2.840.114350.1.13.71.2.7.2.808267", ], "has_data": False, }, @@ -52,10 +56,12 @@ def denormalize_codes(self, schema, cursor): "filter_priority": True, "code_systems": [ "http://terminology.hl7.org/CodeSystem/service-type", - "urn:oid:2.16.840.1.113883.4.642.3.518", "http://snomed.info/sct", - "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/34", - "urn:oid:1.2.840.114350.1.13.71.2.7.10", + # Cerner specific systems + "https://fhir.cerner.com/%/codeSet/34", + # EPIC specific systems + "urn:oid:2.16.840.1.113883.4.642.3.518", + "urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.18886", ], "has_data": False, }, @@ -66,8 +72,10 @@ def denormalize_codes(self, schema, cursor): "code_systems": [ "http://terminology.hl7.org/CodeSystem/v3-ActPriority", "http://snomed.info/sct", - "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/codeSet/3", - "urn:oid:1.2.840.114350.1.13.71.2.7.10", + # Cerner specific systems + "https://fhir.cerner.com/%/codeSet/3", + # EPIC specific systems + "urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.410", ], "has_data": False, }, @@ -80,8 +88,10 @@ def denormalize_codes(self, schema, cursor): "http://snomed.info/sct", "http://hl7.org/fhir/sid/icd-10-cm", "http://hl7.org/fhir/sid/icd-9-cm", - "https://fhir.cerner.com/96976f07-eccb-424c-9825-e0d0b887148b/nomenclature", - "urn:oid:1.2.840.114350.1.13.71.2.7.2", + # Cerner specific systems + "https://fhir.cerner.com/%/nomenclature", + # EPIC specific systems + "urn:oid:1.2.840.114350.1.13.71.2.7.2.728286", ], "has_data": False, }, diff --git a/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja b/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja index 37a1c60f..a0ac8b92 100644 --- a/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja +++ b/cumulus_library/template_sql/codeable_concept_denormalize.sql.jinja @@ -21,7 +21,7 @@ CREATE TABLE {{ target_table }} AS ( {%- endif %} {%- if filter_priority %} WHERE - u.codeable_concept.system LIKE '{{ system }}%' + u.codeable_concept.system LIKE '{{ system }}' {%- endif %} ), --noqa: LT07 {%- endfor %} From 4a4e4521b94a459d7a725a944e30eea7be7b1905 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Thu, 8 Feb 2024 14:19:28 -0500 Subject: [PATCH 3/5] test update --- tests/test_templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index d4ad4877..a546ecb2 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -77,7 +77,7 @@ def test_codeable_concept_denormalize_filter_creation(): source AS s, UNNEST(s.code_col.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system LIKE 'http://snomed.info/sct%' + u.codeable_concept.system LIKE 'http://snomed.info/sct' ), --noqa: LT07 system_code_col_1 AS ( @@ -91,7 +91,7 @@ def test_codeable_concept_denormalize_filter_creation(): source AS s, UNNEST(s.code_col.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-10-cm%' + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-10-cm' ), --noqa: LT07 union_table AS ( From 447abdb861a1887ef6df7aebf0fcfd4d577c889a Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Thu, 8 Feb 2024 14:25:40 -0500 Subject: [PATCH 4/5] clarified one code source --- cumulus_library/studies/core/builder_condition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cumulus_library/studies/core/builder_condition.py b/cumulus_library/studies/core/builder_condition.py index 0ffa52cf..1e005122 100644 --- a/cumulus_library/studies/core/builder_condition.py +++ b/cumulus_library/studies/core/builder_condition.py @@ -55,7 +55,6 @@ def denormalize_codes(self): "http://hl7.org/fhir/sid/icd-10-cm", "http://hl7.org/fhir/sid/icd-9-cm", "http://hl7.org/fhir/sid/icd-9-cm/diagnosis", - # Cerner specific systems "http://terminology.hl7.org/CodeSystem/data-absent-reason", # EPIC specific systems "urn:oid:1.2.840.114350.1.13.71.2.7.2.728286", From afb8eb703bea59b75f38a1005e2c63b08d802492 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Thu, 8 Feb 2024 14:27:43 -0500 Subject: [PATCH 5/5] Reorg order on conditions --- cumulus_library/studies/core/builder_condition.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cumulus_library/studies/core/builder_condition.py b/cumulus_library/studies/core/builder_condition.py index 1e005122..0602973c 100644 --- a/cumulus_library/studies/core/builder_condition.py +++ b/cumulus_library/studies/core/builder_condition.py @@ -55,10 +55,11 @@ def denormalize_codes(self): "http://hl7.org/fhir/sid/icd-10-cm", "http://hl7.org/fhir/sid/icd-9-cm", "http://hl7.org/fhir/sid/icd-9-cm/diagnosis", - "http://terminology.hl7.org/CodeSystem/data-absent-reason", # EPIC specific systems "urn:oid:1.2.840.114350.1.13.71.2.7.2.728286", "urn:oid:1.2.840.114350.1.13.71.2.7.4.698084.10375", + # Spec allowed code of last resort + "http://terminology.hl7.org/CodeSystem/data-absent-reason", ], ) self.queries.append(