Merge pull request #41 from FHIR-Aggregator/fix/upgrade

upgraded fhir.resources version closes #28

Author: teslajoy

README.md

@@ -26,57 +26,46 @@ To run the transformer, ensure that [CDA](https://cda.readthedocs.io/en/latest/)
 Usage: cda2fhir transform [OPTIONS]
 
 Options:
-  -s, --save               Save FHIR ndjson to CDA2FHIR/data/META folder.
-                           [default: True]
+  -s, --save                 Save FHIR ndjson to CDA2FHIR/data/META folder.
+                             [default: True]
   -v, --verbose
-  -ns, --n_samples TEXT    Number of samples to randomly select - max 100.
-  -nd, --n_diagnosis TEXT  Number of diagnosis to randomly select - max 100.
-  -nf, --n_files TEXT      Number of files to randomly select - max 100.
-  -f, --transform_files    Transform CDA files to FHIR DocumentReference and Group.
-  -p, --path TEXT          Path to save the FHIR NDJSON files. default is
-                           CDA2FHIR/data/META.
-  --help                   Show this message and exit.
+  -ns, --n_samples TEXT      Number of samples to randomly select - max 100.
+  -nd, --n_diagnosis TEXT    Number of diagnosis to randomly select - max 100.
+  -nf, --n_files TEXT        Number of files to randomly select - max 100.
+  -f, --transform_files      Transform CDA files to FHIR DocumentReference and
+                             Group.
+  -t, --transform_treatment  Transform CDA treatment to all sub-hierarchy of
+                             FHIR MedicationAdministration ->
+                             SubstanceDefinitionRepresentation.
+  -c, --transform_condition  Transform CDA disease to Condition
+  -m, --transform_mutation   Transform CDA mutation to Observation
+  -p, --path TEXT            Path to save the FHIR NDJSON files. default is
+                             CDA2FHIR/data/META.
+  --help                     Show this message and exit.
 ``` 
 
 - example 
 ``` 
 cda2fhir transform 
 ``` 
 
-NOTE: in-case of interest in validating your FHIR data with GEN3, you will need to go through the [user-guide, setup, and documentation of GEN3 tracker](https://aced-idp.github.io/requirements/) before running the ```cda2fhir``` commands.
-
 ### FHIR data validation 
 
-#### disable gen3-client
-```
-mv ~/.gen3/gen3_client_config.ini ~/.gen3/gen3_client_config.ini-xxx
-mv ~/.gen3/gen3-client ~/.gen3/gen3-client-xxx
-```
-
 #### Run validate
 ```
- time cda2fhir validate
-{'summary': {'Specimen': 721837, 'Observation': 731005, 'ResearchStudy': 423, 'BodyStructure': 163, 'Condition': 95262, 'ResearchSubject': 160649, 'Patient': 138738}}
-
-real    5m
-user    5m
-sys     0m5.1s
-
+ cda2fhir validate --path data/META
+{'summary': {'Specimen': 742505, 'Medication': 214, 'Observation': 832864, 'ResearchStudy': 429, 'SubstanceDefinition': 214, 'BodyStructure': 135, 'Condition': 114804, 'ResearchSubject': 184888, 'MedicationAdministration': 38267, 'Patient': 159047, 'Substance': 214}}
 ```
 
-#### Restore gen3-client
-
-```
-mv ~/.gen3/gen3-client-xxx ~/.gen3/gen3-client
-mv ~/.gen3/gen3_client_config.ini-xxx ~/.gen3/gen3_client_config.ini
-  
-```
-
-
 This command will validate your FHIR entities and their reference relations to each other. It will also generate a summary count of all entities in each ndjson file. 
 
 NOTE: This process may take _**5 minutes**_ or more, depending on your platform or compute power due to the size of the current data.
 
+#### Check for a field ex. extension
+```bash
+awk '!/extension/ {exit 1}' data/META/ResearchSubject.ndjson && echo "Every line contains 'extension'" || echo "Not every line contains 'extension'"
+```
+
 
 ### Testing
 Current integration testing runs on all data and may take approximately _**2 hours**_.

cda2fhir/__init__.py

@@ -0,0 +1,5 @@
+from cda2fhir.patcher import apply_patches
+
+apply_patches()
+
+

cda2fhir/cda2fhir.py

@@ -41,16 +41,17 @@ class CDASubject(Base):
     vital_status: Mapped[Optional[str]] = mapped_column(String)
     days_to_death: Mapped[Optional[int]] = mapped_column(Integer)
     cause_of_death: Mapped[Optional[str]] = mapped_column(String)
-    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer)
+    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer, unique=True)
     researchsubject_subjects: Mapped[List["CDASubjectResearchSubject"]] = relationship(
         back_populates="subject"
     )
-    subject_alias_relation: Mapped[List["CDASubjectAlias"]] = relationship(
-        back_populates="subject_alias_relations"
-    )
     subject_project_relation: Mapped[List["CDASubjectProject"]] = relationship(
         back_populates="subject"
     )
+    subject_identifier: Mapped[List["CDASubjectIdentifier"]] = relationship(
+        "CDASubjectIdentifier",
+        back_populates="subject"
+    )
     subject_file_relation: Mapped[List["CDAFileSubject"]] = relationship(
         back_populates="subject"
     )
@@ -59,14 +60,21 @@ class CDASubject(Base):
         back_populates="subject"
     )
 
-
     @property
     def alias_id(self):
-        """Fetch CDA subject's alias id from subject_alias_relations table - (one to one)"""
-        if self.subject_alias_relation and len(self.subject_alias_relation) > 0:
-            return self.subject_alias_relation.__getitem__(0).subject_alias
-        else:
-            return None
+        return self.integer_id_alias
+
+
+
+class CDASubjectIdentifier(Base):
+    __tablename__ = 'subject_identifier'
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    subject_alias: Mapped[int] = mapped_column(Integer, ForeignKey('subject.integer_id_alias'), nullable=False)
+    system: Mapped[str] = mapped_column(String, nullable=False)
+    field_name: Mapped[str] = mapped_column(String, nullable=False)
+    value: Mapped[str] = mapped_column(String, nullable=False)
+    subject: Mapped["CDASubject"] = relationship("CDASubject", back_populates="subject_identifier")
+
 
 
 class CDAResearchSubject(Base):
@@ -76,7 +84,7 @@ class CDAResearchSubject(Base):
     member_of_research_project: Mapped[Optional[str]] = mapped_column(String)
     primary_diagnosis_condition: Mapped[Optional[str]] = mapped_column(String)
     primary_diagnosis_site: Mapped[Optional[str]] = mapped_column(String)
-    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer)
+    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer, unique=True)
     subject_researchsubjects: Mapped[List["CDASubjectResearchSubject"]] = relationship(
         back_populates="researchsubject"
     )
@@ -94,8 +102,8 @@ class CDAResearchSubject(Base):
 class CDASubjectResearchSubject(Base):
     __tablename__ = 'subject_researchsubject'
     query: QueryPropertyDescriptor = Session.query_property()
-    subject_id: Mapped[str] = mapped_column(ForeignKey("subject.id"), primary_key=True)
-    researchsubject_id: Mapped[str] = mapped_column(ForeignKey("researchsubject.id"), primary_key=True)
+    subject_alias: Mapped[int] = mapped_column(ForeignKey("subject.integer_id_alias"), primary_key=True)
+    researchsubject_alias: Mapped[int] = mapped_column(ForeignKey("researchsubject.integer_id_alias"), primary_key=True)
     subject: Mapped["CDASubject"] = relationship(
         back_populates="researchsubject_subjects"
     )
@@ -104,37 +112,35 @@ class CDASubjectResearchSubject(Base):
     )
 
 
-class CDASubjectAlias(Base):
-    __tablename__ = 'subject_alias_table'
-    query: QueryPropertyDescriptor = Session.query_property()
-    subject_id: Mapped[str] = mapped_column(ForeignKey("subject.id"), primary_key=True)
-    subject_alias: Mapped[int] = mapped_column(Integer, primary_key=True)
-    subject_alias_relations: Mapped["CDASubject"] = relationship(
-        back_populates="subject_alias_relation"
-    )
-
-
 class CDASubjectProject(Base):
     __tablename__ = 'subject_project'
     query: QueryPropertyDescriptor = Session.query_property()
-    subject_id: Mapped[str] = mapped_column(ForeignKey("subject.id"), primary_key=True)
+    subject_alias: Mapped[int] = mapped_column(ForeignKey("subject.integer_id_alias"), primary_key=True)
     associated_project: Mapped[str] = mapped_column(String, primary_key=True)
     subject: Mapped["CDASubject"] = relationship(
         back_populates="subject_project_relation"
     )
 
 
+
 class CDADiagnosis(Base):
     __tablename__ = 'diagnosis'
     query: QueryPropertyDescriptor = Session.query_property()
     id: Mapped[str] = mapped_column(String, primary_key=True)
     primary_diagnosis: Mapped[Optional[str]] = mapped_column(String)
     age_at_diagnosis: Mapped[Optional[int]] = mapped_column(Integer)
     morphology: Mapped[Optional[str]] = mapped_column(String)
-    stage: Mapped[Optional[str]] = mapped_column(String)
+    pathologic_stage: Mapped[Optional[str]] = mapped_column(String)
+    pathologic_stage_m: Mapped[Optional[str]] = mapped_column(String)
+    pathologic_stage_n: Mapped[Optional[str]] = mapped_column(String)
+    pathologic_stage_t: Mapped[Optional[str]] = mapped_column(String)
+    clinical_stage: Mapped[Optional[str]] = mapped_column(String)
+    clinical_stage_m: Mapped[Optional[str]] = mapped_column(String)
+    clinical_stage_n: Mapped[Optional[str]] = mapped_column(String)
+    clinical_stage_t: Mapped[Optional[str]] = mapped_column(String)
     grade: Mapped[Optional[str]] = mapped_column(String)
     method_of_diagnosis: Mapped[Optional[str]] = mapped_column(String)
-    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer)
+    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer, unique=True)
     researchsubject_diagnoses: Mapped[List["CDAResearchSubjectDiagnosis"]] = relationship(
         back_populates="diagnosis"
     )
@@ -143,8 +149,8 @@ class CDADiagnosis(Base):
 class CDAResearchSubjectDiagnosis(Base):
     __tablename__ = 'researchsubject_diagnosis'
     query: QueryPropertyDescriptor = Session.query_property()
-    researchsubject_id: Mapped[str] = mapped_column(ForeignKey('researchsubject.id'), primary_key=True)
-    diagnosis_id: Mapped[str] = mapped_column(ForeignKey('diagnosis.id'), primary_key=True)
+    researchsubject_alias: Mapped[str] = mapped_column(ForeignKey('researchsubject.integer_id_alias'), primary_key=True)
+    diagnosis_alias: Mapped[str] = mapped_column(ForeignKey('diagnosis.integer_id_alias'), primary_key=True)
     diagnosis: Mapped["CDADiagnosis"] = relationship(
         back_populates="researchsubject_diagnoses"
     )
@@ -166,7 +172,7 @@ class CDATreatment(Base):
     treatment_effect: Mapped[Optional[str]] = mapped_column(String)
     treatment_end_reason: Mapped[Optional[str]] = mapped_column(String)
     number_of_cycles: Mapped[Optional[int]] = mapped_column(Integer)
-    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer)
+    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer, unique=True)
     researchsubject_treatments: Mapped[List["CDAResearchSubjectTreatment"]] = relationship(
         back_populates="treatment"
     )
@@ -175,8 +181,8 @@ class CDATreatment(Base):
 class CDAResearchSubjectTreatment(Base):
     __tablename__ = 'researchsubject_treatment'
     query: QueryPropertyDescriptor = Session.query_property()
-    researchsubject_id: Mapped[str] = mapped_column(ForeignKey('researchsubject.id'), primary_key=True)
-    treatment_id: Mapped[str] = mapped_column(ForeignKey('treatment.id'), primary_key=True)
+    researchsubject_alias: Mapped[str] = mapped_column(ForeignKey('researchsubject.integer_id_alias'), primary_key=True)
+    treatment_alias: Mapped[str] = mapped_column(ForeignKey('treatment.integer_id_alias'), primary_key=True)
     researchsubject: Mapped["CDAResearchSubject"] = relationship(
         back_populates="treatment_researchsubjects"
     )
@@ -208,8 +214,8 @@ class CDASpecimen(Base):
 class CDAResearchSubjectSpecimen(Base):
     __tablename__ = 'researchsubject_specimen'
     query: QueryPropertyDescriptor = Session.query_property()
-    researchsubject_id: Mapped[str] = mapped_column(ForeignKey('researchsubject.id'), primary_key=True)
-    specimen_id: Mapped[str] = mapped_column(ForeignKey('specimen.id'), primary_key=True)
+    researchsubject_alias: Mapped[str] = mapped_column(ForeignKey('researchsubject.integer_id_alias'), primary_key=True)
+    specimen_alias: Mapped[str] = mapped_column(ForeignKey('specimen.integer_id_alias'), primary_key=True)
     specimen: Mapped["CDASpecimen"] = relationship(
         back_populates="researchsubject_specimens"
     )
@@ -237,14 +243,6 @@ class GDCProgramdbGap(Base):
 #    query: QueryPropertyDescriptor = Session.query_property()
 #    # TODO: make one table via all xlsx sheets
 
-class CDASubjectIdentifier(Base):
-    __tablename__ = 'cda_subject_identifier' # CDA provenance info relation table.
-    query: QueryPropertyDescriptor = Session.query_property()
-    subject_alias:  Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
-    value: Mapped[Optional[str]] = mapped_column(String, primary_key=True)
-    system: Mapped[Optional[str]] = mapped_column(String, primary_key=True)
-    field_name: Mapped[Optional[str]] = mapped_column(String, primary_key=True)
-
 
 class CDAProjectRelation(Base):
     __tablename__ = 'project_program_relation'
@@ -279,7 +277,7 @@ class CDAFile(Base):
     imaging_modality: Mapped[Optional[str]] = mapped_column(String)
     dbgap_accession_number: Mapped[Optional[str]] = mapped_column(String)
     imaging_series: Mapped[Optional[str]] = mapped_column(String)
-    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer)
+    integer_id_alias: Mapped[Optional[int]] = mapped_column(Integer, unique=True)
     specimen_file_relation: Mapped[List["CDAFileSpecimen"]] = relationship(
         back_populates="file"
     )
@@ -288,11 +286,12 @@ class CDAFile(Base):
     )
 
 
+
 class CDAFileSubject(Base):
     __tablename__ = 'file_subject'
     query: QueryPropertyDescriptor = Session.query_property()
-    file_id: Mapped[str] = mapped_column(ForeignKey("cda_file.id"), primary_key=True)
-    subject_id: Mapped[str] = mapped_column(ForeignKey("subject.id"), primary_key=True)
+    file_alias: Mapped[str] = mapped_column(ForeignKey("cda_file.integer_id_alias"), primary_key=True)
+    subject_alias: Mapped[str] = mapped_column(ForeignKey("subject.integer_id_alias"), primary_key=True)
     subject: Mapped["CDASubject"] = relationship(
         back_populates="subject_file_relation"
     )
@@ -301,11 +300,12 @@ class CDAFileSubject(Base):
     )
 
 
+
 class CDAFileSpecimen(Base):
     __tablename__ = 'file_specimen'
     query: QueryPropertyDescriptor = Session.query_property()
-    file_id: Mapped[str] = mapped_column(ForeignKey("cda_file.id"), primary_key=True)
-    specimen_id: Mapped[str] = mapped_column(ForeignKey("specimen.id"), primary_key=True)
+    file_alias: Mapped[str] = mapped_column(ForeignKey("cda_file.integer_id_alias"), primary_key=True)
+    specimen_alias: Mapped[str] = mapped_column(ForeignKey("specimen.integer_id_alias"), primary_key=True)
     specimen: Mapped["CDASpecimen"] = relationship(
         back_populates="file_specimen_relation"
     )