From d9a6297b561047fae356c4c93c49ecef654a7425 Mon Sep 17 00:00:00 2001 From: Vincent Janvid <69356202+Vince-janv@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:56:56 +0100 Subject: [PATCH] Fix(Alembic) Remove imports from cg in alembic scripts (#2986)(patch) ### Fixed - Removed all import from cg in the alembic scripts --- ...1_25_d241d8c493fb_add_pipeline_to_order.py | 22 +++++++++- ..._de0f5b78dca4_rename_pipeline_sars_cov2.py | 44 +++++++++++++++++-- ...ix_sars_cov_2_data_analysis_to_database.py | 26 +++++++++-- ...8e54d17f4f3_add_is_clinical_to_customer.py | 13 +++++- ...a5065b4_add_taxprofiler_analysis_option.py | 22 +++++++++- ...72b_add_raredisease_to_analysis_options.py | 29 ++++++++++-- ...998be2e367cf_fix_mip_on_fastq_wgs_cases.py | 33 +++++++++++--- .../c3da223e60d8_add_has_backup_field.py | 30 +++++++++++-- .../c3fdf3a8a5b3_add_data_archive_location.py | 13 +++++- ...d21feaa0_add_pipeline_limitations_table.py | 24 +++++++++- .../e9df15a35de4_fix_tumour_not_to_maf.py | 34 +++++++++++--- 11 files changed, 260 insertions(+), 30 deletions(-) diff --git a/alembic/versions/2024_01_25_d241d8c493fb_add_pipeline_to_order.py b/alembic/versions/2024_01_25_d241d8c493fb_add_pipeline_to_order.py index 76bf5cf2ce..781cafe78c 100644 --- a/alembic/versions/2024_01_25_d241d8c493fb_add_pipeline_to_order.py +++ b/alembic/versions/2024_01_25_d241d8c493fb_add_pipeline_to_order.py @@ -6,10 +6,11 @@ """ +from enum import StrEnum + import sqlalchemy as sa from alembic import op -from cg.constants import Workflow # revision identifiers, used by Alembic. revision = "d241d8c493fb" @@ -18,6 +19,25 @@ depends_on = None +class Workflow(StrEnum): + BALSAMIC: str = "balsamic" + BALSAMIC_QC: str = "balsamic-qc" + BALSAMIC_UMI: str = "balsamic-umi" + BALSAMIC_PON: str = "balsamic-pon" + DEMULTIPLEX: str = "demultiplex" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + RAREDISEASE: str = "raredisease" + RNAFUSION: str = "rnafusion" + RSYNC: str = "rsync" + SARS_COV_2: str = "sars-cov-2" + SPRING: str = "spring" + TAXPROFILER: str = "taxprofiler" + + def upgrade(): op.add_column( table_name="order", diff --git a/alembic/versions/2024_01_25_de0f5b78dca4_rename_pipeline_sars_cov2.py b/alembic/versions/2024_01_25_de0f5b78dca4_rename_pipeline_sars_cov2.py index 6935f1ae58..38a6810b8d 100644 --- a/alembic/versions/2024_01_25_de0f5b78dca4_rename_pipeline_sars_cov2.py +++ b/alembic/versions/2024_01_25_de0f5b78dca4_rename_pipeline_sars_cov2.py @@ -6,12 +6,33 @@ """ -from sqlalchemy import orm +from enum import StrEnum + +from sqlalchemy import orm, types from sqlalchemy.dialects import mysql +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from alembic import op -from cg.constants import Workflow -from cg.store.models import Analysis, Case + + +class Workflow(StrEnum): + BALSAMIC: str = "balsamic" + BALSAMIC_PON: str = "balsamic-pon" + BALSAMIC_QC: str = "balsamic-qc" + BALSAMIC_UMI: str = "balsamic-umi" + DEMULTIPLEX: str = "demultiplex" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + MUTANT: str = "mutant" + RAREDISEASE: str = "raredisease" + RNAFUSION: str = "rnafusion" + RSYNC: str = "rsync" + SPRING: str = "spring" + TAXPROFILER: str = "taxprofiler" + # revision identifiers, used by Alembic. revision = "de0f5b78dca4" @@ -44,6 +65,23 @@ new_enum = mysql.ENUM(*list(new_options)) +class Base(DeclarativeBase): + pass + + +class Analysis(Base): + __tablename__ = "analysis" + + id: Mapped[int] = mapped_column(primary_key=True) + pipeline: Mapped[str] + + +class Case(Base): + __tablename__ = "case" + data_analysis: Mapped[str] + id: Mapped[int] = mapped_column(primary_key=True) + + def upgrade(): bind = op.get_bind() session = orm.Session(bind=bind) diff --git a/alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py b/alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py index 28010199ec..8546331426 100644 --- a/alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py +++ b/alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py @@ -6,13 +6,14 @@ """ +from enum import StrEnum + import sqlalchemy as sa from sqlalchemy import Column, orm, types from sqlalchemy.dialects import mysql from sqlalchemy.orm import declarative_base from alembic import op -from cg.constants import DataDelivery, Workflow Base = declarative_base() @@ -23,13 +24,32 @@ depends_on = None +class DataDelivery(StrEnum): + ANALYSIS_FILES: str = "analysis" + ANALYSIS_BAM_FILES: str = "analysis-bam" + FASTQ: str = "fastq" + NIPT_VIEWER: str = "nipt-viewer" + FASTQ_QC: str = "fastq_qc" + SCOUT: str = "scout" + + +class Pipeline(StrEnum): + BALSAMIC: str = "balsamic" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + SARS_COV_2: str = "sars-cov-2" + + class Case(Base): __tablename__ = "family" id = sa.Column(sa.types.Integer, primary_key=True) internal_id = sa.Column(sa.types.String(32), unique=True, nullable=False) name = sa.Column(sa.types.String(128), nullable=False) - data_analysis = Column(types.Enum(*list(Workflow))) + data_analysis = Column(types.Enum(*list(Pipeline))) data_delivery = Column(types.Enum(*list(DataDelivery))) def __str__(self) -> str: @@ -58,7 +78,7 @@ def upgrade(): .filter(Case.data_analysis == "") ): print(f"Altering family: {str(family)}") - family.data_analysis = str(Workflow.SARS_COV_2) + family.data_analysis = str(Pipeline.SARS_COV_2) print(f"Altered family: {str(family)}") session.commit() diff --git a/alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py b/alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py index 7d9c52a63a..cd49cf14d6 100644 --- a/alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py +++ b/alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py @@ -7,9 +7,10 @@ """ import sqlalchemy as sa +from sqlalchemy import Column, types +from sqlalchemy.orm import DeclarativeBase from alembic import op -from cg.store.models import Customer # revision identifiers, used by Alembic. revision = "68e54d17f4f3" @@ -18,6 +19,16 @@ depends_on = None +class Model(DeclarativeBase): + pass + + +class Customer(Model): + __tablename__ = "customer" + id = Column(types.Integer, primary_key=True) + is_clinical = Column(types.Boolean, nullable=True) + + def upgrade(): op.add_column(table_name="customer", column=sa.Column("is_clinical", sa.BOOLEAN)) bind = op.get_bind() diff --git a/alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py b/alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py index e5ffbc80c6..716c3b07c3 100644 --- a/alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py +++ b/alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py @@ -6,12 +6,13 @@ """ +from enum import StrEnum + import sqlalchemy as sa from sqlalchemy.dialects import mysql from sqlalchemy.orm import declarative_base from alembic import op -from cg.constants import Workflow # revision identifiers, used by Alembic. revision = "9008aa5065b4" @@ -43,6 +44,25 @@ new_enum = mysql.ENUM(*new_options) +class Workflow(StrEnum): + BALSAMIC: str = "balsamic" + BALSAMIC_PON: str = "balsamic-pon" + BALSAMIC_QC: str = "balsamic-qc" + BALSAMIC_UMI: str = "balsamic-umi" + DEMULTIPLEX: str = "demultiplex" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + MUTANT: str = "mutant" + RAREDISEASE: str = "raredisease" + RNAFUSION: str = "rnafusion" + RSYNC: str = "rsync" + SPRING: str = "spring" + TAXPROFILER: str = "taxprofiler" + + class Analysis(Base): __tablename__ = "analysis" id = sa.Column(sa.types.Integer, primary_key=True) diff --git a/alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py b/alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py index a89749e91e..3ce3127999 100644 --- a/alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py +++ b/alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py @@ -6,12 +6,13 @@ """ +from enum import StrEnum + import sqlalchemy as sa from sqlalchemy.dialects import mysql from sqlalchemy.orm import declarative_base from alembic import op -from cg.constants import Workflow # revision identifiers, used by Alembic. revision = "9073c61bc72b" @@ -38,6 +39,27 @@ "spring", "taxprofiler", ) + + +class Pipeline(StrEnum): + BALSAMIC: str = "balsamic" + BALSAMIC_QC: str = "balsamic-qc" + BALSAMIC_UMI: str = "balsamic-umi" + BALSAMIC_PON: str = "balsamic-pon" + DEMULTIPLEX: str = "demultiplex" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + RAREDISEASE: str = "raredisease" + RNAFUSION: str = "rnafusion" + RSYNC: str = "rsync" + SARS_COV_2: str = "sars-cov-2" + SPRING: str = "spring" + TAXPROFILER: str = "taxprofiler" + + new_analysis_options = sorted(old_analysis_options + ("raredisease",)) old_analysis_enum = mysql.ENUM(*old_analysis_options) @@ -47,13 +69,14 @@ class Analysis(Base): __tablename__ = "analysis" id = sa.Column(sa.types.Integer, primary_key=True) - pipeline = sa.Column(sa.types.Enum(*list(Workflow))) + pipeline = sa.Column(sa.types.Enum(*list(Pipeline))) class Case(Base): __tablename__ = "family" id = sa.Column(sa.types.Integer, primary_key=True) - data_analysis = sa.Column(sa.types.Enum(*list(Workflow))) + data_analysis = sa.Column(sa.types.Enum(*list(Pipeline))) + internal_id = sa.Column(sa.types.String) def upgrade(): diff --git a/alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py b/alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py index 2bc9ab979a..7fcfecb937 100644 --- a/alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py +++ b/alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py @@ -7,13 +7,13 @@ """ from datetime import datetime +from enum import StrEnum import sqlalchemy as sa from sqlalchemy import orm from sqlalchemy.orm import declarative_base from alembic import op -from cg.constants import PREP_CATEGORIES, DataDelivery, Workflow Base = declarative_base() @@ -23,6 +23,27 @@ branch_labels = None depends_on = None +PREP_CATEGORIES = ("cov", "mic", "rml", "tgs", "wes", "wgs", "wts") + + +class DataDelivery(StrEnum): + ANALYSIS_FILES: str = "analysis" + ANALYSIS_BAM_FILES: str = "analysis-bam" + FASTQ: str = "fastq" + NIPT_VIEWER: str = "nipt-viewer" + FASTQ_QC: str = "fastq_qc" + SCOUT: str = "scout" + + +class Pipeline(StrEnum): + BALSAMIC: str = "balsamic" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + SARS_COV_2: str = "sars-cov-2" + class Customer(Base): __tablename__ = "customer" @@ -38,7 +59,7 @@ class Case(Base): name = sa.Column(sa.types.String(128), nullable=False) customer_id = sa.Column(sa.ForeignKey("customer.id", ondelete="CASCADE"), nullable=False) customer = orm.relationship(Customer, foreign_keys=[customer_id]) - data_analysis = sa.Column(sa.types.Enum(*list(Workflow))) + data_analysis = sa.Column(sa.types.Enum(*list(Pipeline))) data_delivery = sa.Column(sa.types.Enum(*list(DataDelivery))) priority = sa.Column(sa.types.Integer, default=1, nullable=False) _panels = sa.Column(sa.types.Text) @@ -111,7 +132,7 @@ def upgrade(): session.query(Case) .filter(Case.customer_id == 1) .filter(Case.data_delivery == DataDelivery.FASTQ) - .filter(Case.data_analysis == Workflow.FASTQ) + .filter(Case.data_analysis == Pipeline.FASTQ) .filter(Case.priority == "research") .filter(Case.ordered_at >= datetime(year=2021, month=2, day=2)) ): @@ -126,7 +147,7 @@ def upgrade(): and sample.name == family.name ): print(f"changing data analysis from FASTQ to MIP for: {family}") - family.data_analysis = Workflow.MIP_DNA + family.data_analysis = Pipeline.MIP_DNA count += 1 session.commit() @@ -143,7 +164,7 @@ def downgrade(): session.query(Case) .filter(Case.customer_id == 1) .filter(Case.data_delivery == DataDelivery.FASTQ) - .filter(Case.data_analysis == Workflow.MIP_DNA) + .filter(Case.data_analysis == Pipeline.MIP_DNA) .filter(Case.priority == "research") .filter(Case.ordered_at >= datetime(year=2021, month=2, day=2)) ): @@ -158,7 +179,7 @@ def downgrade(): and sample.name == family.name ): print(f"changing data analysis from MIP to FASTQ for: {family}") - family.data_analysis = Workflow.FASTQ + family.data_analysis = Pipeline.FASTQ count += 1 session.commit() diff --git a/alembic/versions/c3da223e60d8_add_has_backup_field.py b/alembic/versions/c3da223e60d8_add_has_backup_field.py index bb12490866..294fd9f2ed 100644 --- a/alembic/versions/c3da223e60d8_add_has_backup_field.py +++ b/alembic/versions/c3da223e60d8_add_has_backup_field.py @@ -6,12 +6,13 @@ """ +from enum import StrEnum + import sqlalchemy as sa -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Column, types +from sqlalchemy.orm import DeclarativeBase, sessionmaker from alembic import op -from cg.constants.sequencing import Sequencers -from cg.store.models import Flowcell # revision identifiers, used by Alembic. revision = "c3da223e60d8" @@ -22,6 +23,29 @@ Session = sessionmaker() +class Model(DeclarativeBase): + pass + + +class Flowcell(Model): + __tablename__ = "flowcell" + id = Column(types.Integer, primary_key=True) + sequencer_type = Column(types.Enum("hiseqga", "hiseqx", "novaseq", "novaseqx")) + sequenced_at = Column(types.DateTime) + has_backup = Column(types.Boolean, nullable=False, default=False) + + +class Sequencers(StrEnum): + """Sequencer instruments.""" + + ALL: str = "all" + HISEQX: str = "hiseqx" + HISEQGA: str = "hiseqga" + NOVASEQ: str = "novaseq" + NOVASEQX: str = "novaseqx" + OTHER: str = "other" + + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### bind = op.get_bind() diff --git a/alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py b/alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py index 36628fceb7..37e2800f55 100644 --- a/alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py +++ b/alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py @@ -7,9 +7,10 @@ """ import sqlalchemy as sa +from sqlalchemy import Column, types +from sqlalchemy.orm import DeclarativeBase from alembic import op -from cg.store.models import Customer # revision identifiers, used by Alembic. revision = "c3fdf3a8a5b3" @@ -18,6 +19,16 @@ depends_on = None +class Model(DeclarativeBase): + pass + + +class Customer(Model): + __tablename__ = "customer" + id = Column(types.Integer, primary_key=True) + data_archive_location = Column(types.String(32), nullable=False, default="PDC") + + def upgrade(): op.add_column(table_name="customer", column=sa.Column("data_archive_location", sa.String(32))) bind = op.get_bind() diff --git a/alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py b/alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py index 4f0cb6b7d0..4a6239033b 100644 --- a/alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py +++ b/alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py @@ -6,10 +6,11 @@ """ +from enum import StrEnum + import sqlalchemy as sa from alembic import op -from cg.constants.constants import Workflow # revision identifiers, used by Alembic. revision = "e853d21feaa0" @@ -20,6 +21,25 @@ table_name = "application_limitations" +class Pipeline(StrEnum): + BALSAMIC: str = "balsamic" + BALSAMIC_QC: str = "balsamic-qc" + BALSAMIC_UMI: str = "balsamic-umi" + BALSAMIC_PON: str = "balsamic-pon" + DEMULTIPLEX: str = "demultiplex" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + RAREDISEASE: str = "raredisease" + RNAFUSION: str = "rnafusion" + RSYNC: str = "rsync" + SARS_COV_2: str = "sars-cov-2" + SPRING: str = "spring" + TAXPROFILER: str = "taxprofiler" + + def upgrade(): op.create_table( table_name, @@ -30,7 +50,7 @@ def upgrade(): sa.ForeignKey("application.id", ondelete="CASCADE"), nullable=False, ), - sa.Column("pipeline", sa.Enum(*list(Workflow)), nullable=False), + sa.Column("pipeline", sa.Enum(*list(Pipeline)), nullable=False), sa.Column("limitations", sa.Text()), sa.Column("comment", sa.Text()), sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()), diff --git a/alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py b/alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py index 54a5c1659f..650f364ba6 100644 --- a/alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py +++ b/alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py @@ -7,13 +7,13 @@ """ from datetime import datetime +from enum import StrEnum import sqlalchemy as sa from sqlalchemy import orm from sqlalchemy.orm import declarative_base from alembic import op -from cg.constants import PREP_CATEGORIES, DataDelivery, Workflow # revision identifiers, used by Alembic. revision = "e9df15a35de4" @@ -24,6 +24,28 @@ Base = declarative_base() +class Pipeline(StrEnum): + BALSAMIC: str = "balsamic" + FASTQ: str = "fastq" + FLUFFY: str = "fluffy" + MICROSALT: str = "microsalt" + MIP_DNA: str = "mip-dna" + MIP_RNA: str = "mip-rna" + SARS_COV_2: str = "sars-cov-2" + + +class DataDelivery(StrEnum): + ANALYSIS_FILES: str = "analysis" + ANALYSIS_BAM_FILES: str = "analysis-bam" + FASTQ: str = "fastq" + NIPT_VIEWER: str = "nipt-viewer" + FASTQ_QC: str = "fastq_qc" + SCOUT: str = "scout" + + +PREP_CATEGORIES = ("cov", "mic", "rml", "tgs", "wes", "wgs", "wts") + + class Customer(Base): __tablename__ = "customer" @@ -38,7 +60,7 @@ class Case(Base): name = sa.Column(sa.types.String(128), nullable=False) customer_id = sa.Column(sa.ForeignKey("customer.id", ondelete="CASCADE"), nullable=False) customer = orm.relationship(Customer, foreign_keys=[customer_id]) - data_analysis = sa.Column(sa.types.Enum(*list(Workflow))) + data_analysis = sa.Column(sa.types.Enum(*list(Pipeline))) data_delivery = sa.Column(sa.types.Enum(*list(DataDelivery))) priority = sa.Column(sa.types.Integer, default=1, nullable=False) _panels = sa.Column(sa.types.Text) @@ -112,7 +134,7 @@ def upgrade(): session.query(Case) .filter(Case.customer_id == 1) .filter(Case.data_delivery == DataDelivery.FASTQ) - .filter(Case.data_analysis == Workflow.MIP_DNA) + .filter(Case.data_analysis == Pipeline.MIP_DNA) .filter(Case.priority == "research") ): if len(family.links) > 1: @@ -130,7 +152,7 @@ def upgrade(): and sample.name == family.name ): print(f"changing data analysis from MIP to FASTQ for: {family}") - family.data_analysis = Workflow.FASTQ + family.data_analysis = Pipeline.FASTQ count += 1 session.commit() @@ -146,7 +168,7 @@ def downgrade(): session.query(Case) .filter(Case.customer_id == 1) .filter(Case.data_delivery == DataDelivery.FASTQ) - .filter(Case.data_analysis == Workflow.FASTQ) + .filter(Case.data_analysis == Pipeline.FASTQ) .filter(Case.priority == "research") ): if len(family.links) > 1: @@ -164,7 +186,7 @@ def downgrade(): and sample.name == family.name ): print(f"changing data analysis from FASTQ to MIP-DNA for: {family}") - family.data_analysis = Workflow.MIP_DNA + family.data_analysis = Pipeline.MIP_DNA count += 1 session.commit()