fix: fix RxNorm product to ingredient mappings (#72)

* chore: code format export sql files
* fix: RxNorm mappings via RxNav paths
* chore: proper capitalization in flake.nix
* feat: add cli, changelog
* chore: move HTML path table
* chore: delete fix-ingredients
* chore: bump package version

Author: zietzm

CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## [3.1.0] - 2025-05-08
+
+### Added
+
+- Correct RxNorm paths between products and ingredients (per RxNav)
+- Command line interface for generating the ZIP archive
+
+### Fixed
+
+- Erroneous product-to-ingredient mapping that led to products having thousands of ingredients

README.md

@@ -197,11 +197,10 @@ Simple answer?
 
 ### Releasing
 
-Here's how the v3.0.0 archive was created.
+Here's how the v3.1.0 archive was created.
 
 ```bash
-cd database
-zip -r ../onsides-v3.0.0.zip annotations csv database_scripts schema
+build-zip --version v3.1.0
 ```
 
 ---

flake.nix

@@ -40,7 +40,7 @@
             test_tabula() {
               python -c "import tabula" >/dev/null 2>&1
               if [ $? -eq 0 ]; then
-                echo "Onsides development environment activated!"
+                echo "OnSIDES development environment activated!"
               else
                 echo "There was an error installing tabula. Please check your installation."
               fi

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "onsides"
-version = "3.0.0"
+version = "3.1.0"
 description = "Database of drug adverse events from product labels"
 readme = "README.md"
 authors = [
@@ -12,6 +12,7 @@ requires-python = ">=3.12"
 dependencies = [
     "beautifulsoup4>=4.13.3",
     "connectorx>=0.4.2",
+    "duckdb>=1.2.2",
     "fastexcel>=0.13.0",
     "httpx>=0.28.1",
     "lxml>=5.3.1",
@@ -41,3 +42,6 @@ build-backend = "hatchling.build"
 dev = [
     "pytest>=8.3.5",
 ]
+
+[project.scripts]
+build-zip = "onsides.cli:main"

snakemake/onsides/export/Snakefile

@@ -5,6 +5,7 @@ import polars as pl
 from rich.progress import track
 
 from onsides import rxnorm
+from onsides.rxnorm_ingredients import generate_rxnorm_map
 
 tables = [
     "product_adverse_effect",
@@ -178,16 +179,26 @@ rule combine_results_to_sqlite:
             input.us_sql,
             input.uk_sql,
             input.eu_sql,
+        ]
+        for file in duckdb_scripts:
+            print(f"Running: 'duckdb duck.db < {file}'")
+            shell("duckdb duck.db < {file}")
+
+        generate_rxnorm_map("snakemake/onsides/export/rxnav-path-table.html")
+
+        vocab_scripts = [
             input.vocab_sql,
             input.filter_sql,
         ]
-        for file in duckdb_scripts:
+        for file in vocab_scripts:
+            print(f"Running: 'duckdb duck.db < {file}'")
             shell("duckdb duck.db < {file}")
 
         sqlite_scripts = [
             input.threshold_sql,
         ]
         for file in sqlite_scripts:
+            print(f"Running: 'sqlite3 database/onsides.db < {file}'")
             shell("sqlite3 database/onsides.db < {file}")
 
 

snakemake/onsides/export/eu_mapping.sql

@@ -1,43 +1,43 @@
-create table eu_preds as
-select
-    text_id as effect_meddra_id,
-    split_part(text_id, '.', 1) as source_product_id,
+CREATE TABLE eu_preds AS
+SELECT
+    text_id AS effect_meddra_id,
+    split_part(text_id, '.', 1) AS source_product_id,
     term_id,
     pred0,
     pred1
-from
+FROM
     '_onsides/combined/label_english_preds.parquet'
-where
+WHERE
     ends_with(text_id, 'EU');
 
-create table eu_meta as
-select
-    name as source_product_name,
-    code as source_product_id,
-    page_url as source_label_url
-from
+CREATE TABLE eu_meta AS
+SELECT
+    name AS source_product_name,
+    code AS source_product_id,
+    page_url AS source_label_url
+FROM
     '_onsides/eu/label_text.parquet';
 
-create table eu_rxnorm as
-select
-    code as source_product_id,
-    rxcui as rxnorm_product_id
-from
+CREATE TABLE eu_rxnorm AS
+SELECT
+    code AS source_product_id,
+    rxcui AS rxnorm_product_id
+FROM
     '_onsides/eu/labels_to_rxnorm.parquet';
 
-create table eu_final as
-select
+CREATE TABLE eu_final AS
+SELECT
     source_product_name,
     source_product_id,
     source_label_url,
     term_id,
     rxnorm_product_id,
     pred0,
     pred1
-from
+FROM
     eu_preds
-    inner join eu_meta using (source_product_id)
-    inner join eu_rxnorm using (source_product_id);
+    INNER JOIN eu_meta USING (source_product_id)
+    INNER JOIN eu_rxnorm USING (source_product_id);
 
 --------------------------------------------------------------------------------
 --------------------------------------------------------------------------------
@@ -49,57 +49,53 @@ LOAD sqlite;
 ATTACH IF NOT EXISTS 'database/onsides.db' AS db (TYPE sqlite);
 
 -- Product label
-insert into
+INSERT INTO
     db.product_label (
         source,
         source_product_name,
         source_product_id,
         source_label_url
+    ) WITH eu_inner AS (
+        SELECT
+            DISTINCT source_product_name,
+            source_product_id,
+            source_label_url
+        FROM
+            eu_final
     )
-with
-eu_inner as (
-    select distinct
-        source_product_name,
-        source_product_id,
-        source_label_url
-    from
-        eu_final
-)
-select
-    'EU' as source,
+SELECT
+    'EU' AS source,
     source_product_name,
     source_product_id,
     source_label_url
-from
+FROM
     eu_inner;
 
 -- Product to RxNorm
 INSERT INTO
-    db.product_to_rxnorm (label_id, rxnorm_product_id)
-WITH
-new_labels AS (
-    SELECT
-        label_id,
-        source_product_id
-    FROM
-        db.product_label
-    WHERE
-        source = 'EU'
-),
-joined_labels as (
-    SELECT DISTINCT
-        label_id,
-        rxnorm_product_id
-    FROM
-        new_labels
-        INNER JOIN eu_final USING (source_product_id)
-)
-select
+    db.product_to_rxnorm (label_id, rxnorm_product_id) WITH new_labels AS (
+        SELECT
+            label_id,
+            source_product_id
+        FROM
+            db.product_label
+        WHERE
+            source = 'EU'
+    ),
+    joined_labels AS (
+        SELECT
+            DISTINCT label_id,
+            rxnorm_product_id
+        FROM
+            new_labels
+            INNER JOIN eu_final USING (source_product_id)
+    )
+SELECT
     label_id,
     rxnorm_product_id
-from
+FROM
     joined_labels
-where
+WHERE
     rxnorm_product_id != '';
 
 -- Adverse effect
@@ -111,20 +107,18 @@ INSERT INTO
         match_method,
         pred0,
         pred1
+    ) WITH new_labels AS (
+        SELECT
+            label_id,
+            source_product_id
+        FROM
+            db.product_label
+        WHERE
+            source = 'EU'
     )
-WITH
-new_labels AS (
-    SELECT
-        label_id,
-        source_product_id
-    FROM
-        db.product_label
-    WHERE
-        source = 'EU'
-)
 SELECT
     label_id AS product_label_id,
-    'NA' as label_section,
+    'NA' AS label_section,
     term_id AS effect_meddra_id,
     'PMB' AS match_method,
     pred0,