From 227ba45532b102473f8014e384451f720f9035b5 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Wed, 19 Jun 2024 21:44:19 +0200 Subject: [PATCH 01/30] feat[wip]: add ml inference pipeline --- .../versions/bb3f9a1b3fa2_add_ml_bad_lumis.py | 42 +++++ etl/python/env.py | 1 + etl/python/models/__init__.py | 2 + etl/python/models/ml_bad_lumis.py | 21 +++ .../pipelines/file_ingesting/pipeline.py | 55 +++++++ etl/python/pipelines/ml_inference/__init__.py | 0 etl/python/pipelines/ml_inference/extract.py | 27 ++++ etl/python/pipelines/ml_inference/pipeline.py | 64 ++++++++ etl/python/pipelines/ml_inference/predict.py | 15 ++ .../pipelines/ml_inference/preprocess.py | 10 ++ poetry.lock | 148 +++++++++++++++++- pyproject.toml | 1 + 12 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py create mode 100644 etl/python/models/ml_bad_lumis.py create mode 100644 etl/python/pipelines/ml_inference/__init__.py create mode 100644 etl/python/pipelines/ml_inference/extract.py create mode 100644 etl/python/pipelines/ml_inference/pipeline.py create mode 100644 etl/python/pipelines/ml_inference/predict.py create mode 100644 etl/python/pipelines/ml_inference/preprocess.py diff --git a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py new file mode 100644 index 00000000..81d80a20 --- /dev/null +++ b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py @@ -0,0 +1,42 @@ +# noqa: INP001 + +"""add ml bad lumis + +Revision ID: bb3f9a1b3fa2 +Revises: 86e3beee4a68 +Create Date: 2024-03-26 16:09:50.366283 + +""" + +from collections.abc import Sequence + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "bb3f9a1b3fa2" +down_revision: str = "86e3beee4a68" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def fact_ml_bad_lumis() -> list: + op.execute(""" + CREATE TABLE IF NOT EXISTS fact_ml_bad_lumis ( + model_name VARCHAR(255), + dataset_id BIGINT, + file_id BIGINT, + run_number INT, + ls_number INT, + me_id INT, + CONSTRAINT fact_ml_bad_lumis_pk PRIMARY KEY (model_name, dataset_id, run_number, ls_number, me_id) + ); + """) + + +def upgrade(engine_name: str) -> None: + fact_ml_bad_lumis() + + +def downgrade(engine_name: str) -> None: + op.drop_table("fact_ml_bad_lumis") diff --git a/etl/python/env.py b/etl/python/env.py index 24e80b9c..d7f9bee6 100644 --- a/etl/python/env.py +++ b/etl/python/env.py @@ -4,6 +4,7 @@ app_env = config("ENV") eos_landing_zone = config("EOS_LANDING_ZONE") mounted_eos_path = config("MOUNTED_EOS_PATH", default=None) +model_registry_path = config("MODEL_REGISTRY_PATH") conn_str = config("DATABASE_URI") lxplus_user = config("KEYTAB_USER") lxplus_pwd = config("KEYTAB_PWD") diff --git a/etl/python/models/__init__.py b/etl/python/models/__init__.py index a94704df..b28777f7 100644 --- a/etl/python/models/__init__.py +++ b/etl/python/models/__init__.py @@ -2,6 +2,7 @@ from .dim_mes import DimMonitoringElements from .file_index import FactFileIndex from .lumisection import FactLumisection +from .ml_bad_lumis import FactMLBadLumis from .run import FactRun from .th1 import FactTH1 from .th2 import FactTH2 @@ -15,4 +16,5 @@ "FactLumisection", "FactTH1", "FactTH2", + "FactMLBadLumis", ] diff --git a/etl/python/models/ml_bad_lumis.py b/etl/python/models/ml_bad_lumis.py new file mode 100644 index 00000000..26fb7371 --- /dev/null +++ b/etl/python/models/ml_bad_lumis.py @@ -0,0 +1,21 @@ +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base + + +Base = declarative_base() + + +class FactMLBadLumis(Base): + __tablename__ = "fact_ml_bad_lumis" + + model_name = sa.Column("model_name", sa.String(length=255)) + dataset_id = sa.Column("dataset_id", sa.BigInteger) + file_id = sa.Column("file_id", sa.BigInteger) + run_number = sa.Column("run_number", sa.Integer) + ls_number = sa.Column("ls_number", sa.Integer) + me_id = sa.Column("me_id", sa.Integer) + + __table_args__ = ( + sa.PrimaryKeyConstraint("model_name", "dataset_id", "run_number", "ls_number", "me_id"), + sa.Index("idx_fmbl_model_name_run_number", "model_name", "run_number"), + ) diff --git a/etl/python/pipelines/file_ingesting/pipeline.py b/etl/python/pipelines/file_ingesting/pipeline.py index 745f18ac..0d73efcf 100644 --- a/etl/python/pipelines/file_ingesting/pipeline.py +++ b/etl/python/pipelines/file_ingesting/pipeline.py @@ -4,6 +4,7 @@ from ...env import conn_str from ...models.file_index import StatusCollection +from ..ml_inference.pipeline import pipeline as ml_pipeline from ..utils import clean_file, error_handler from .exceptions import PipelineCopyError, PipelineRootfileError from .extract import extract @@ -13,6 +14,46 @@ from .utils import validate_root_file +WORKSPACES_WITH_ML = { + "jetmet": [ + { + "file": "model_CHFrac_highPt_Barrel_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_highPt_Barrel", + "thr": 0.05, + }, + { + "file": "model_CHFrac_highPt_EndCap_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_highPt_EndCap", + "thr": 0.05, + }, + { + "file": "model_CHFrac_lowPt_Barrel_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_lowPt_Barrel", + "thr": 0.05, + }, + { + "file": "model_CHFrac_lowPt_EndCap_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_lowPt_EndCap", + "thr": 0.05, + }, + { + "file": "model_CHFrac_mediumPt_Barrel_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_mediumPt_Barrel", + "thr": 0.05, + }, + { + "file": "model_CHFrac_mediumPt_EndCap_checkpoint_20240517.onnx", + "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_mediumPt_EndCap", + "thr": 0.05, + }, + {"file": "model_MET_2_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/MET_2", "thr": 0.05}, + {"file": "model_METPhi_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/METPhi", "thr": 0.05}, + {"file": "model_METSig_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/METSig", "thr": 0.05}, + {"file": "model_SumET_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/SumET", "thr": 0.05}, + ] +} + + def pipeline(workspace_name: str, workspace_mes: str, file_id: int, dataset_id: int): """ Note: always re-raise exceptions to mark the task as failed in celery broker @@ -47,4 +88,18 @@ def pipeline(workspace_name: str, workspace_mes: str, file_id: int, dataset_id: # If everything goes well, we can clean the file clean_file(fpath) + + # Run ML pipeline for each model if workspace has any models registered + if workspace_name in WORKSPACES_WITH_ML: + for model in WORKSPACES_WITH_ML[workspace_name]: + ml_pipeline( + workspace_name=workspace_name, + model_file=model["file"], + model_thr=model["thr"], + model_me=model["me"], + dataset_id=dataset_id, + file_id=file_id, + ) + + # Finally finishes post_load(engine, file_id) diff --git a/etl/python/pipelines/ml_inference/__init__.py b/etl/python/pipelines/ml_inference/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/etl/python/pipelines/ml_inference/extract.py b/etl/python/pipelines/ml_inference/extract.py new file mode 100644 index 00000000..bb752cf7 --- /dev/null +++ b/etl/python/pipelines/ml_inference/extract.py @@ -0,0 +1,27 @@ +from sqlalchemy.engine.base import Engine +from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import sessionmaker + +from ...models import DimMonitoringElements, FactTH1, FactTH2 + + +def extract_me(engine: Engine, me: str): + sess = sessionmaker(bind=engine) + with sess() as session: + query = session.query(DimMonitoringElements).filter(DimMonitoringElements.me == me) + try: + result = query.one() + except NoResultFound: + result = None + return result + + +def extract(engine: Engine, th_class: FactTH1 | FactTH2, dataset_id: int, file_id: int, me_id: int): + sess = sessionmaker(bind=engine) + with sess() as session: + query = session.query(th_class).filter( + th_class.dataset_id == dataset_id, + th_class.file_id == file_id, + th_class.me_id == me_id, + ) + return query.all() diff --git a/etl/python/pipelines/ml_inference/pipeline.py b/etl/python/pipelines/ml_inference/pipeline.py new file mode 100644 index 00000000..9c53455e --- /dev/null +++ b/etl/python/pipelines/ml_inference/pipeline.py @@ -0,0 +1,64 @@ +import os.path + +import pandas as pd +from sqlalchemy import create_engine + +from ...common.pgsql import copy_expert +from ...env import conn_str +from ...models import FactMLBadLumis, FactTH1, FactTH2 +from .extract import extract, extract_me +from .predict import predict +from .preprocess import preprocess + + +def pipeline( + workspace_name: str, + model_file: str, + model_thr: float, + model_me: str, + dataset_id: int, + file_id: int, +): + engine = create_engine(f"{conn_str}/{workspace_name}") + + # Extrac me_id and TH dimension if me exists in database + me = extract_me(engine, model_me) + if me is None: + return + + # Extract data + th_class = FactTH1 if me.dim == 1 else FactTH2 + hists = extract(engine, th_class, dataset_id, file_id, me.me_id) + if len(hists) == 0: + return + + # Preprocess data + lss_, input_data = preprocess(hists) + + # Predictions + preds = predict(workspace_name, model_file, input_data) + + # Select bad lumis + model_name = os.path.splitext(model_file)[0] + bad_lumis = [] + for idx, ls_number in enumerate(lss_.flatten()): + mse = preds[1][idx] + if mse >= model_thr: + bad_lumis.append( + { + "model_name": model_name, + "dataset_id": dataset_id, + "file_id": file_id, + "run_number": hists[idx].run_number, + "ls_number": ls_number, + "me_id": me.me_id, + } + ) + + if len(bad_lumis) == 0: + return + + # Dump bad lumis if there is any + bad_lumis = pd.DataFrame(bad_lumis) + bad_lumis.to_sql(name=FactMLBadLumis.__tablename__, con=engine, if_exists="append", index=False, method=copy_expert) + engine.dispose() diff --git a/etl/python/pipelines/ml_inference/predict.py b/etl/python/pipelines/ml_inference/predict.py new file mode 100644 index 00000000..cdb3c99c --- /dev/null +++ b/etl/python/pipelines/ml_inference/predict.py @@ -0,0 +1,15 @@ +import numpy as np +from onnxruntime import InferenceSession + +from ...env import model_registry_path + + +def predict(workspace_name: str, model_file: str, input_data: np.array) -> list[dict]: + model_path = f"{model_registry_path}/{workspace_name}/{model_file}" + sess = InferenceSession(model_path) + + # Predict + input_name = sess.get_inputs()[0].name + result = sess.run(None, {input_name: input_data}) + + return result diff --git a/etl/python/pipelines/ml_inference/preprocess.py b/etl/python/pipelines/ml_inference/preprocess.py new file mode 100644 index 00000000..70a2664d --- /dev/null +++ b/etl/python/pipelines/ml_inference/preprocess.py @@ -0,0 +1,10 @@ +import numpy as np + + +def preprocess(data: list[dict]) -> tuple: + results_ = [{"ls_number": result.ls_number, "data": result.data} for result in data] + sorted_ = sorted(results_, key=lambda x: x["ls_number"]) + test_array = np.vstack([histogram["data"] for histogram in sorted_]) + test_array = test_array.astype(np.float32) + lss_ = np.vstack([histogram["ls_number"] for histogram in sorted_]) + return lss_, test_array diff --git a/poetry.lock b/poetry.lock index 50044af1..c0b4236e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -442,6 +442,23 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +description = "Colored terminal output for Python's logging module" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, + {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, +] + +[package.dependencies] +humanfriendly = ">=9.1" + +[package.extras] +cron = ["capturer (>=2.4)"] + [[package]] name = "contourpy" version = "1.2.0" @@ -756,6 +773,17 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] typing = ["typing-extensions (>=4.8)"] +[[package]] +name = "flatbuffers" +version = "24.3.25" +description = "The FlatBuffers serialization format for Python" +optional = false +python-versions = "*" +files = [ + {file = "flatbuffers-24.3.25-py2.py3-none-any.whl", hash = "sha256:8dbdec58f935f3765e4f7f3cf635ac3a77f83568138d6a2311f524ec96364812"}, + {file = "flatbuffers-24.3.25.tar.gz", hash = "sha256:de2ec5b203f21441716617f38443e0a8ebf3d25bf0d9c0bb0ce68fa00ad546a4"}, +] + [[package]] name = "flower" version = "2.0.1" @@ -931,6 +959,20 @@ setproctitle = ["setproctitle"] testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] tornado = ["tornado (>=0.2)"] +[[package]] +name = "humanfriendly" +version = "10.0" +description = "Human friendly output for text interfaces using Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] + +[package.dependencies] +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} + [[package]] name = "humanize" version = "4.9.0" @@ -1277,6 +1319,23 @@ pillow = ">=8" pyparsing = ">=2.3.1" python-dateutil = ">=2.7" +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + [[package]] name = "nodeenv" version = "1.8.0" @@ -1336,6 +1395,48 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "onnxruntime" +version = "1.18.0" +description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +optional = false +python-versions = "*" +files = [ + {file = "onnxruntime-1.18.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:5a3b7993a5ecf4a90f35542a4757e29b2d653da3efe06cdd3164b91167bbe10d"}, + {file = "onnxruntime-1.18.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15b944623b2cdfe7f7945690bfb71c10a4531b51997c8320b84e7b0bb59af902"}, + {file = "onnxruntime-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e61ce5005118064b1a0ed73ebe936bc773a102f067db34108ea6c64dd62a179"}, + {file = "onnxruntime-1.18.0-cp310-cp310-win32.whl", hash = "sha256:a4fc8a2a526eb442317d280610936a9f73deece06c7d5a91e51570860802b93f"}, + {file = "onnxruntime-1.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:71ed219b768cab004e5cd83e702590734f968679bf93aa488c1a7ffbe6e220c3"}, + {file = "onnxruntime-1.18.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:3d24bd623872a72a7fe2f51c103e20fcca2acfa35d48f2accd6be1ec8633d960"}, + {file = "onnxruntime-1.18.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f15e41ca9b307a12550bfd2ec93f88905d9fba12bab7e578f05138ad0ae10d7b"}, + {file = "onnxruntime-1.18.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f45ca2887f62a7b847d526965686b2923efa72538c89b7703c7b3fe970afd59"}, + {file = "onnxruntime-1.18.0-cp311-cp311-win32.whl", hash = "sha256:9e24d9ecc8781323d9e2eeda019b4b24babc4d624e7d53f61b1fe1a929b0511a"}, + {file = "onnxruntime-1.18.0-cp311-cp311-win_amd64.whl", hash = "sha256:f8608398976ed18aef450d83777ff6f77d0b64eced1ed07a985e1a7db8ea3771"}, + {file = "onnxruntime-1.18.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f1d79941f15fc40b1ee67738b2ca26b23e0181bf0070b5fb2984f0988734698f"}, + {file = "onnxruntime-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e8caf3a8565c853a22d323a3eebc2a81e3de7591981f085a4f74f7a60aab2d"}, + {file = "onnxruntime-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:498d2b8380635f5e6ebc50ec1b45f181588927280f32390fb910301d234f97b8"}, + {file = "onnxruntime-1.18.0-cp312-cp312-win32.whl", hash = "sha256:ba7cc0ce2798a386c082aaa6289ff7e9bedc3dee622eef10e74830cff200a72e"}, + {file = "onnxruntime-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:1fa175bd43f610465d5787ae06050c81f7ce09da2bf3e914eb282cb8eab363ef"}, + {file = "onnxruntime-1.18.0-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:0284c579c20ec8b1b472dd190290a040cc68b6caec790edb960f065d15cf164a"}, + {file = "onnxruntime-1.18.0-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d47353d036d8c380558a5643ea5f7964d9d259d31c86865bad9162c3e916d1f6"}, + {file = "onnxruntime-1.18.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:885509d2b9ba4b01f08f7fa28d31ee54b6477953451c7ccf124a84625f07c803"}, + {file = "onnxruntime-1.18.0-cp38-cp38-win32.whl", hash = "sha256:8614733de3695656411d71fc2f39333170df5da6c7efd6072a59962c0bc7055c"}, + {file = "onnxruntime-1.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:47af3f803752fce23ea790fd8d130a47b2b940629f03193f780818622e856e7a"}, + {file = "onnxruntime-1.18.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:9153eb2b4d5bbab764d0aea17adadffcfc18d89b957ad191b1c3650b9930c59f"}, + {file = "onnxruntime-1.18.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c7fd86eca727c989bb8d9c5104f3c45f7ee45f445cc75579ebe55d6b99dfd7c"}, + {file = "onnxruntime-1.18.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac67a4de9c1326c4d87bcbfb652c923039b8a2446bb28516219236bec3b494f5"}, + {file = "onnxruntime-1.18.0-cp39-cp39-win32.whl", hash = "sha256:6ffb445816d06497df7a6dd424b20e0b2c39639e01e7fe210e247b82d15a23b9"}, + {file = "onnxruntime-1.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:46de6031cb6745f33f7eca9e51ab73e8c66037fb7a3b6b4560887c5b55ab5d5d"}, +] + +[package.dependencies] +coloredlogs = "*" +flatbuffers = "*" +numpy = ">=1.21.6" +packaging = "*" +protobuf = "*" +sympy = "*" + [[package]] name = "packaging" version = "23.2" @@ -1617,6 +1718,26 @@ files = [ [package.dependencies] wcwidth = "*" +[[package]] +name = "protobuf" +version = "5.27.0" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-5.27.0-cp310-abi3-win32.whl", hash = "sha256:2f83bf341d925650d550b8932b71763321d782529ac0eaf278f5242f513cc04e"}, + {file = "protobuf-5.27.0-cp310-abi3-win_amd64.whl", hash = "sha256:b276e3f477ea1eebff3c2e1515136cfcff5ac14519c45f9b4aa2f6a87ea627c4"}, + {file = "protobuf-5.27.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:744489f77c29174328d32f8921566fb0f7080a2f064c5137b9d6f4b790f9e0c1"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:f51f33d305e18646f03acfdb343aac15b8115235af98bc9f844bf9446573827b"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:56937f97ae0dcf4e220ff2abb1456c51a334144c9960b23597f044ce99c29c89"}, + {file = "protobuf-5.27.0-cp38-cp38-win32.whl", hash = "sha256:a17f4d664ea868102feaa30a674542255f9f4bf835d943d588440d1f49a3ed15"}, + {file = "protobuf-5.27.0-cp38-cp38-win_amd64.whl", hash = "sha256:aabbbcf794fbb4c692ff14ce06780a66d04758435717107c387f12fb477bf0d8"}, + {file = "protobuf-5.27.0-cp39-cp39-win32.whl", hash = "sha256:587be23f1212da7a14a6c65fd61995f8ef35779d4aea9e36aad81f5f3b80aec5"}, + {file = "protobuf-5.27.0-cp39-cp39-win_amd64.whl", hash = "sha256:7cb65fc8fba680b27cf7a07678084c6e68ee13cab7cace734954c25a43da6d0f"}, + {file = "protobuf-5.27.0-py3-none-any.whl", hash = "sha256:673ad60f1536b394b4fa0bcd3146a4130fcad85bfe3b60eaa86d6a0ace0fa374"}, + {file = "protobuf-5.27.0.tar.gz", hash = "sha256:07f2b9a15255e3cf3f137d884af7972407b556a7a220912b252f26dc3121e6bf"}, +] + [[package]] name = "psutil" version = "5.9.8" @@ -1777,6 +1898,17 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pyreadline3" +version = "3.4.1" +description = "A python implementation of GNU readline." +optional = false +python-versions = "*" +files = [ + {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, + {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, +] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -2115,6 +2247,20 @@ files = [ dev = ["build", "hatch"] doc = ["sphinx"] +[[package]] +name = "sympy" +version = "1.12.1" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sympy-1.12.1-py3-none-any.whl", hash = "sha256:9b2cbc7f1a640289430e13d2a56f02f867a1da0190f2f99d8968c2f74da0e515"}, + {file = "sympy-1.12.1.tar.gz", hash = "sha256:2877b03f998cd8c08f07cd0de5b767119cd3ef40d09f41c30d722f6686b0fb88"}, +] + +[package.dependencies] +mpmath = ">=1.1.0,<1.4.0" + [[package]] name = "tenacity" version = "8.2.3" @@ -2255,4 +2401,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a07cc44374dc8182ab8f53ef5f76326c569399a14e0755098f0899ec7cf96228" +content-hash = "c4d333e64e26e285b9013e490e47f653b969f076b4e7c33be4662b2ef322b104" diff --git a/pyproject.toml b/pyproject.toml index fc52e4ef..4299b28f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ pandas = "^2.2.1" paramiko = "^3.4.0" scp = "^0.14.5" flower = "^2.0.1" +onnxruntime = "^1.18.0" [tool.poetry.group.dev.dependencies] pre-commit = "^3.6.0" From 82a2e62c2176bac58278c580168ad242b92a563c Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 25 Jun 2024 15:08:58 +0200 Subject: [PATCH 02/30] feat: create ml models index table in alembic migrations and add sqlalchemy model --- .../versions/bb3f9a1b3fa2_add_ml_bad_lumis.py | 16 +++++++++++++++ etl/python/models/__init__.py | 2 ++ etl/python/models/dim_ml_index.py | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 etl/python/models/dim_ml_index.py diff --git a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py index 81d80a20..e88d747b 100644 --- a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py +++ b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py @@ -34,9 +34,25 @@ def fact_ml_bad_lumis() -> list: """) +def dim_ml_models_index() -> list: + op.execute(""" + CREATE TABLE IF NOT EXISTS dim_ml_models_index ( + model_id SERIAL, + filename VARCHAR(255), + target_me VARCHAR(255), + thr DOUBLE PRECISION, + active BOOLEAN, + CONSTRAINT dim_ml_models_index_pk PRIMARY KEY (model_id) + ); + """) + op.execute("CREATE INDEX idx_active ON dim_ml_models_index (active);") + + def upgrade(engine_name: str) -> None: + dim_ml_models_index() fact_ml_bad_lumis() def downgrade(engine_name: str) -> None: + op.drop_table("dim_ml_models_index") op.drop_table("fact_ml_bad_lumis") diff --git a/etl/python/models/__init__.py b/etl/python/models/__init__.py index b28777f7..31ba9a3f 100644 --- a/etl/python/models/__init__.py +++ b/etl/python/models/__init__.py @@ -1,5 +1,6 @@ from .dataset_index import FactDatasetIndex from .dim_mes import DimMonitoringElements +from .dim_ml_index import DimMLModelsIndex from .file_index import FactFileIndex from .lumisection import FactLumisection from .ml_bad_lumis import FactMLBadLumis @@ -9,6 +10,7 @@ __all__ = [ + "DimMLModelsIndex", "DimMonitoringElements", "FactDatasetIndex", "FactFileIndex", diff --git a/etl/python/models/dim_ml_index.py b/etl/python/models/dim_ml_index.py new file mode 100644 index 00000000..441b0610 --- /dev/null +++ b/etl/python/models/dim_ml_index.py @@ -0,0 +1,20 @@ +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base + + +Base = declarative_base() + + +class DimMLModelsIndex(Base): + __tablename__ = "dim_ml_models_index" + + model_id = sa.Column("model_id", sa.BigInteger, autoincrement=True) + filename = sa.Column("filename", sa.String(255)) + target_me = sa.Column("target_me", sa.String(255)) + thr = sa.Column("thr", sa.Float) + active = sa.Column("active", sa.Boolean) + + __table_args__ = ( + sa.PrimaryKeyConstraint("model_id"), + sa.Index("idx_active", "active"), + ) From e741d67f76c99b65fdf678d0875b91fc8c34e682 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 25 Jun 2024 15:10:17 +0200 Subject: [PATCH 03/30] refactor: replace model_name with model_id in fact_ml_bad_lumis table --- etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py | 5 +++-- etl/python/models/ml_bad_lumis.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py index e88d747b..2d3c3af8 100644 --- a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py +++ b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py @@ -21,15 +21,16 @@ def fact_ml_bad_lumis() -> list: + # We don't need extra indexes op.execute(""" CREATE TABLE IF NOT EXISTS fact_ml_bad_lumis ( - model_name VARCHAR(255), + model_id BIGINT, dataset_id BIGINT, file_id BIGINT, run_number INT, ls_number INT, me_id INT, - CONSTRAINT fact_ml_bad_lumis_pk PRIMARY KEY (model_name, dataset_id, run_number, ls_number, me_id) + CONSTRAINT fact_ml_bad_lumis_pk PRIMARY KEY (model_id, dataset_id, run_number, ls_number, me_id) ); """) diff --git a/etl/python/models/ml_bad_lumis.py b/etl/python/models/ml_bad_lumis.py index 26fb7371..de41c374 100644 --- a/etl/python/models/ml_bad_lumis.py +++ b/etl/python/models/ml_bad_lumis.py @@ -8,7 +8,7 @@ class FactMLBadLumis(Base): __tablename__ = "fact_ml_bad_lumis" - model_name = sa.Column("model_name", sa.String(length=255)) + model_id = sa.Column("model_id", sa.String(length=255)) dataset_id = sa.Column("dataset_id", sa.BigInteger) file_id = sa.Column("file_id", sa.BigInteger) run_number = sa.Column("run_number", sa.Integer) From ad977c15c20dcab83892c55624cba12144ece6be Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 25 Jun 2024 15:12:10 +0200 Subject: [PATCH 04/30] refactor: read active models and metadata from dim_ml_models_index table in workspace's database --- .../pipelines/file_ingesting/pipeline.py | 63 ++++--------------- etl/python/pipelines/file_ingesting/utils.py | 10 +++ etl/python/pipelines/ml_inference/pipeline.py | 14 ++--- 3 files changed, 28 insertions(+), 59 deletions(-) diff --git a/etl/python/pipelines/file_ingesting/pipeline.py b/etl/python/pipelines/file_ingesting/pipeline.py index 0d73efcf..da18b0f0 100644 --- a/etl/python/pipelines/file_ingesting/pipeline.py +++ b/etl/python/pipelines/file_ingesting/pipeline.py @@ -11,47 +11,7 @@ from .post_load import post_load from .pre_extract import pre_extract from .transform_load import transform_load -from .utils import validate_root_file - - -WORKSPACES_WITH_ML = { - "jetmet": [ - { - "file": "model_CHFrac_highPt_Barrel_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_highPt_Barrel", - "thr": 0.05, - }, - { - "file": "model_CHFrac_highPt_EndCap_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_highPt_EndCap", - "thr": 0.05, - }, - { - "file": "model_CHFrac_lowPt_Barrel_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_lowPt_Barrel", - "thr": 0.05, - }, - { - "file": "model_CHFrac_lowPt_EndCap_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_lowPt_EndCap", - "thr": 0.05, - }, - { - "file": "model_CHFrac_mediumPt_Barrel_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_mediumPt_Barrel", - "thr": 0.05, - }, - { - "file": "model_CHFrac_mediumPt_EndCap_checkpoint_20240517.onnx", - "me": "JetMET/Jet/Cleanedak4PFJetsCHS/CHFrac_mediumPt_EndCap", - "thr": 0.05, - }, - {"file": "model_MET_2_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/MET_2", "thr": 0.05}, - {"file": "model_METPhi_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/METPhi", "thr": 0.05}, - {"file": "model_METSig_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/METSig", "thr": 0.05}, - {"file": "model_SumET_checkpoint_20240517.onnx", "me": "JetMET/MET/pfMETT1/Cleaned/SumET", "thr": 0.05}, - ] -} +from .utils import fetch_active_models, validate_root_file def pipeline(workspace_name: str, workspace_mes: str, file_id: int, dataset_id: int): @@ -90,16 +50,17 @@ def pipeline(workspace_name: str, workspace_mes: str, file_id: int, dataset_id: clean_file(fpath) # Run ML pipeline for each model if workspace has any models registered - if workspace_name in WORKSPACES_WITH_ML: - for model in WORKSPACES_WITH_ML[workspace_name]: - ml_pipeline( - workspace_name=workspace_name, - model_file=model["file"], - model_thr=model["thr"], - model_me=model["me"], - dataset_id=dataset_id, - file_id=file_id, - ) + active_models = fetch_active_models(engine) + for model in active_models: + ml_pipeline( + workspace_name=workspace_name, + model_id=model.model_id, + model_file=model.filename, + thr=model.thr, + target_me=model.target_me, + dataset_id=dataset_id, + file_id=file_id, + ) # Finally finishes post_load(engine, file_id) diff --git a/etl/python/pipelines/file_ingesting/utils.py b/etl/python/pipelines/file_ingesting/utils.py index 31ea1502..000fcd71 100644 --- a/etl/python/pipelines/file_ingesting/utils.py +++ b/etl/python/pipelines/file_ingesting/utils.py @@ -1,4 +1,8 @@ import ROOT +from sqlalchemy.engine.base import Engine +from sqlalchemy.orm import sessionmaker + +from ...models import DimMLModelsIndex def validate_root_file(fpath: str) -> None: @@ -8,3 +12,9 @@ def validate_root_file(fpath: str) -> None: """ with ROOT.TFile(fpath) as root_file: root_file.GetUUID().AsString() + + +def fetch_active_models(engine: Engine) -> list[DimMLModelsIndex]: + Session = sessionmaker(bind=engine) # noqa: N806 + with Session() as session: + return session.query(DimMLModelsIndex).filter(DimMLModelsIndex.active).all() diff --git a/etl/python/pipelines/ml_inference/pipeline.py b/etl/python/pipelines/ml_inference/pipeline.py index 9c53455e..126135fb 100644 --- a/etl/python/pipelines/ml_inference/pipeline.py +++ b/etl/python/pipelines/ml_inference/pipeline.py @@ -1,5 +1,3 @@ -import os.path - import pandas as pd from sqlalchemy import create_engine @@ -13,16 +11,17 @@ def pipeline( workspace_name: str, + model_id: int, model_file: str, - model_thr: float, - model_me: str, + thr: float, + target_me: str, dataset_id: int, file_id: int, ): engine = create_engine(f"{conn_str}/{workspace_name}") # Extrac me_id and TH dimension if me exists in database - me = extract_me(engine, model_me) + me = extract_me(engine, target_me) if me is None: return @@ -39,14 +38,13 @@ def pipeline( preds = predict(workspace_name, model_file, input_data) # Select bad lumis - model_name = os.path.splitext(model_file)[0] bad_lumis = [] for idx, ls_number in enumerate(lss_.flatten()): mse = preds[1][idx] - if mse >= model_thr: + if mse >= thr: bad_lumis.append( { - "model_name": model_name, + "model_id": model_id, "dataset_id": dataset_id, "file_id": file_id, "run_number": hists[idx].run_number, From 2f1aabe174191d1ad0c771fe3faacfb1f8842638 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 25 Jun 2024 15:13:06 +0200 Subject: [PATCH 05/30] chore: add `add_ml_model_to_index_handler` to `cli.py` --- etl/cli.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/etl/cli.py b/etl/cli.py index a0688187..c0a97bd8 100755 --- a/etl/cli.py +++ b/etl/cli.py @@ -4,7 +4,7 @@ from python.config import common_indexer_queue, pds_queues, primary_datasets, priority_era, workspaces from python.env import conn_str -from python.models import FactFileIndex, FactTH1, FactTH2 +from python.models import DimMLModelsIndex, FactFileIndex, FactTH1, FactTH2 from python.models.file_index import StatusCollection from python.pipelines.dataset_indexer.tasks import dataset_indexer_pipeline_task from python.pipelines.file_downloader.tasks import file_downloader_pipeline_task @@ -143,6 +143,15 @@ def indexing_handler(args): ) +def add_ml_model_to_index_hanlder(args): + engine = get_engine(args.workspace) + Session = sessionmaker(bind=engine) # noqa: N806 + with Session() as session: + model = DimMLModelsIndex(filename=args.filename, target_me=args.target_me, thr=args.thr, active=args.active) + session.add(model) + session.commit() + + def main(): parser = argparse.ArgumentParser(description="DIALS etl command line interface") subparsers = parser.add_subparsers(dest="command", title="Commands") @@ -191,6 +200,19 @@ def main(): ) clean_table_parser.set_defaults(handler=clean_parsing_error_handler) + # Register ml model command + add_ml_model_parser = subparsers.add_parser("add-ml-model-to-index", help="Register ML molde into DB") + add_ml_model_parser.add_argument("-w", "--workspace", help="Workspace name.", required=True) + add_ml_model_parser.add_argument("-f", "--filename", help="Model binary filename", required=True) + add_ml_model_parser.add_argument( + "-m", "--target-me", help="Monitoring element predicted by the model", required=True + ) + add_ml_model_parser.add_argument( + "-t", "--thr", help="Model threshold for anomaly detection", required=True, type=float + ) + add_ml_model_parser.add_argument("-a", "--active", help="Is the model active?", required=True, type=bool) + add_ml_model_parser.set_defaults(handler=add_ml_model_to_index_hanlder) + args = parser.parse_args() if hasattr(args, "handler"): From cfa8b746980516dc6f5aac409ae19693324ce0e7 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 25 Jun 2024 15:13:17 +0200 Subject: [PATCH 06/30] chore: add ML_models directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 09893315..3a32c2af 100644 --- a/.gitignore +++ b/.gitignore @@ -335,3 +335,4 @@ docker-compose.yaml DQMIO/ usercert.pem userkey.pem +ML_models/ From 3320f90decd52d9599430a1c98dec507fab1441f Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 10:50:18 +0200 Subject: [PATCH 07/30] feat: add index on fact_ml_bad_lumis for dataset_id and run_number --- etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py index 2d3c3af8..8140c964 100644 --- a/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py +++ b/etl/alembic/versions/bb3f9a1b3fa2_add_ml_bad_lumis.py @@ -33,6 +33,7 @@ def fact_ml_bad_lumis() -> list: CONSTRAINT fact_ml_bad_lumis_pk PRIMARY KEY (model_id, dataset_id, run_number, ls_number, me_id) ); """) + op.execute("CREATE INDEX idx_mlbl_dataset_id_run_number ON fact_ml_bad_lumis (dataset_id, run_number);") def dim_ml_models_index() -> list: From bf7eca8e9330df289aef318784e382415e55ed2b Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 10:50:46 +0200 Subject: [PATCH 08/30] fix: FactMLBadLumis sqlalchemy model with incorrect PK and Index --- etl/python/models/ml_bad_lumis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/python/models/ml_bad_lumis.py b/etl/python/models/ml_bad_lumis.py index de41c374..d715ecc0 100644 --- a/etl/python/models/ml_bad_lumis.py +++ b/etl/python/models/ml_bad_lumis.py @@ -16,6 +16,6 @@ class FactMLBadLumis(Base): me_id = sa.Column("me_id", sa.Integer) __table_args__ = ( - sa.PrimaryKeyConstraint("model_name", "dataset_id", "run_number", "ls_number", "me_id"), - sa.Index("idx_fmbl_model_name_run_number", "model_name", "run_number"), + sa.PrimaryKeyConstraint("model_id", "dataset_id", "run_number", "ls_number", "me_id"), + sa.Index("idx_mlbl_dataset_id_run_number", "dataset_id", "run_number"), ) From 7043d4d294ecc132fb56c92678dd1a893befbbf1 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 10:53:49 +0200 Subject: [PATCH 09/30] feat: add ml-models-index endpoint --- backend/dials/settings.py | 1 + backend/dials/urls.py | 2 ++ backend/ml_models_index/__init__.py | 0 backend/ml_models_index/apps.py | 6 ++++ backend/ml_models_index/filters.py | 15 ++++++++++ backend/ml_models_index/models.py | 21 ++++++++++++++ backend/ml_models_index/routers.py | 7 +++++ backend/ml_models_index/serializers.py | 9 ++++++ backend/ml_models_index/viewsets.py | 39 ++++++++++++++++++++++++++ 9 files changed, 100 insertions(+) create mode 100644 backend/ml_models_index/__init__.py create mode 100644 backend/ml_models_index/apps.py create mode 100644 backend/ml_models_index/filters.py create mode 100644 backend/ml_models_index/models.py create mode 100644 backend/ml_models_index/routers.py create mode 100644 backend/ml_models_index/serializers.py create mode 100644 backend/ml_models_index/viewsets.py diff --git a/backend/dials/settings.py b/backend/dials/settings.py index 013c81c0..9da000df 100644 --- a/backend/dials/settings.py +++ b/backend/dials/settings.py @@ -62,6 +62,7 @@ "lumisection.apps.LumisectionConfig", "th1.apps.TH1Config", "th2.apps.TH2Config", + "ml_models_index.apps.MLModelsIndexConfig", "cern_auth.apps.CERNAuthConfig", ] diff --git a/backend/dials/urls.py b/backend/dials/urls.py index 9a334ead..29b848ba 100644 --- a/backend/dials/urls.py +++ b/backend/dials/urls.py @@ -5,6 +5,7 @@ from django.views.generic import TemplateView from file_index.routers import router as file_index_router from lumisection.routers import router as lumisection_router +from ml_models_index.routers import router as ml_models_index_router from rest_framework import routers from run.routers import router as run_router from th1.routers import router as th1_router @@ -19,6 +20,7 @@ router.registry.extend(lumisection_router.registry) router.registry.extend(th1_router.registry) router.registry.extend(th2_router.registry) +router.registry.extend(ml_models_index_router.registry) router.registry.extend(cern_auth_router.registry) swagger_view = TemplateView.as_view(template_name="swagger-ui.html", extra_context={"schema_url": "openapi-schema"}) diff --git a/backend/ml_models_index/__init__.py b/backend/ml_models_index/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/ml_models_index/apps.py b/backend/ml_models_index/apps.py new file mode 100644 index 00000000..601654af --- /dev/null +++ b/backend/ml_models_index/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class MLModelsIndexConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "ml_models_index" diff --git a/backend/ml_models_index/filters.py b/backend/ml_models_index/filters.py new file mode 100644 index 00000000..1a7866a6 --- /dev/null +++ b/backend/ml_models_index/filters.py @@ -0,0 +1,15 @@ +from typing import ClassVar + +from django_filters import rest_framework as filters + +from .models import MLModelsIndex + + +class MLModelsIndexFilter(filters.FilterSet): + class Meta: + model = MLModelsIndex + fields: ClassVar[dict[str, list[str]]] = { + "model_id": ["exact", "in"], + "target_me": ["exact", "regex"], + "active": ["exact"], + } diff --git a/backend/ml_models_index/models.py b/backend/ml_models_index/models.py new file mode 100644 index 00000000..25c5b000 --- /dev/null +++ b/backend/ml_models_index/models.py @@ -0,0 +1,21 @@ +from typing import ClassVar + +from django.db import models + + +class MLModelsIndex(models.Model): + model_id = models.IntegerField(primary_key=True) + filename = models.CharField(max_length=255) + target_me = models.CharField(max_length=255) + thr = models.FloatField() + active = models.BooleanField() + + class Meta: + managed = False + db_table = "dim_ml_models_index" + indexes: ClassVar[list[models.Index]] = [ + models.Index(name="idx_active", fields=["active"]), + ] + + def __str__(self) -> str: + return f"Model <{self.model_id}>" diff --git a/backend/ml_models_index/routers.py b/backend/ml_models_index/routers.py new file mode 100644 index 00000000..791bc711 --- /dev/null +++ b/backend/ml_models_index/routers.py @@ -0,0 +1,7 @@ +from rest_framework import routers + +from .viewsets import MLModelsIndexViewSet + + +router = routers.SimpleRouter() +router.register(r"ml-models-index", MLModelsIndexViewSet, basename="ml-models-index") diff --git a/backend/ml_models_index/serializers.py b/backend/ml_models_index/serializers.py new file mode 100644 index 00000000..15797d04 --- /dev/null +++ b/backend/ml_models_index/serializers.py @@ -0,0 +1,9 @@ +from rest_framework import serializers + +from .models import MLModelsIndex + + +class MLModelsIndexSerializer(serializers.ModelSerializer): + class Meta: + model = MLModelsIndex + fields = "__all__" diff --git a/backend/ml_models_index/viewsets.py b/backend/ml_models_index/viewsets.py new file mode 100644 index 00000000..72536563 --- /dev/null +++ b/backend/ml_models_index/viewsets.py @@ -0,0 +1,39 @@ +import logging +from typing import ClassVar + +from django.conf import settings +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page +from django.views.decorators.vary import vary_on_headers +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import mixins, viewsets +from rest_framework.authentication import BaseAuthentication +from utils.db_router import GenericViewSetRouter +from utils.rest_framework_cern_sso.authentication import ( + CERNKeycloakClientSecretAuthentication, + CERNKeycloakConfidentialAuthentication, +) + +from .filters import MLModelsIndexFilter +from .models import MLModelsIndex +from .serializers import MLModelsIndexSerializer + + +logger = logging.getLogger(__name__) + + +@method_decorator(cache_page(settings.CACHE_TTL), name="retrieve") +@method_decorator(cache_page(settings.CACHE_TTL), name="list") +@method_decorator(vary_on_headers(settings.WORKSPACE_HEADER), name="retrieve") +@method_decorator(vary_on_headers(settings.WORKSPACE_HEADER), name="list") +class MLModelsIndexViewSet( + GenericViewSetRouter, mixins.RetrieveModelMixin, mixins.ListModelMixin, viewsets.GenericViewSet +): + queryset = MLModelsIndex.objects.all().order_by(MLModelsIndex._meta.pk.name) + serializer_class = MLModelsIndexSerializer + filterset_class = MLModelsIndexFilter + filter_backends: ClassVar[list[DjangoFilterBackend]] = [DjangoFilterBackend] + authentication_classes: ClassVar[list[BaseAuthentication]] = [ + CERNKeycloakClientSecretAuthentication, + CERNKeycloakConfidentialAuthentication, + ] From d617a076e3ae9f5915471ddbf2f2e3215deb91fa Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 10:55:18 +0200 Subject: [PATCH 10/30] feat: add ml-bad-lumisection endpoint --- backend/dials/settings.py | 1 + backend/dials/urls.py | 2 + backend/ml_bad_lumisection/__init__.py | 0 backend/ml_bad_lumisection/apps.py | 6 ++ backend/ml_bad_lumisection/filters.py | 18 ++++ backend/ml_bad_lumisection/models.py | 31 ++++++ backend/ml_bad_lumisection/routers.py | 7 ++ backend/ml_bad_lumisection/serializers.py | 9 ++ backend/ml_bad_lumisection/viewsets.py | 118 ++++++++++++++++++++++ 9 files changed, 192 insertions(+) create mode 100644 backend/ml_bad_lumisection/__init__.py create mode 100644 backend/ml_bad_lumisection/apps.py create mode 100644 backend/ml_bad_lumisection/filters.py create mode 100644 backend/ml_bad_lumisection/models.py create mode 100644 backend/ml_bad_lumisection/routers.py create mode 100644 backend/ml_bad_lumisection/serializers.py create mode 100644 backend/ml_bad_lumisection/viewsets.py diff --git a/backend/dials/settings.py b/backend/dials/settings.py index 9da000df..c5840eb6 100644 --- a/backend/dials/settings.py +++ b/backend/dials/settings.py @@ -63,6 +63,7 @@ "th1.apps.TH1Config", "th2.apps.TH2Config", "ml_models_index.apps.MLModelsIndexConfig", + "ml_bad_lumisection.apps.MLBadLumisectionConfig", "cern_auth.apps.CERNAuthConfig", ] diff --git a/backend/dials/urls.py b/backend/dials/urls.py index 29b848ba..409cddf1 100644 --- a/backend/dials/urls.py +++ b/backend/dials/urls.py @@ -5,6 +5,7 @@ from django.views.generic import TemplateView from file_index.routers import router as file_index_router from lumisection.routers import router as lumisection_router +from ml_bad_lumisection.routers import router as ml_bad_lumisection_router from ml_models_index.routers import router as ml_models_index_router from rest_framework import routers from run.routers import router as run_router @@ -21,6 +22,7 @@ router.registry.extend(th1_router.registry) router.registry.extend(th2_router.registry) router.registry.extend(ml_models_index_router.registry) +router.registry.extend(ml_bad_lumisection_router.registry) router.registry.extend(cern_auth_router.registry) swagger_view = TemplateView.as_view(template_name="swagger-ui.html", extra_context={"schema_url": "openapi-schema"}) diff --git a/backend/ml_bad_lumisection/__init__.py b/backend/ml_bad_lumisection/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/ml_bad_lumisection/apps.py b/backend/ml_bad_lumisection/apps.py new file mode 100644 index 00000000..acdbea37 --- /dev/null +++ b/backend/ml_bad_lumisection/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class MLBadLumisectionConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "ml_bad_lumisection" diff --git a/backend/ml_bad_lumisection/filters.py b/backend/ml_bad_lumisection/filters.py new file mode 100644 index 00000000..65857b44 --- /dev/null +++ b/backend/ml_bad_lumisection/filters.py @@ -0,0 +1,18 @@ +from typing import ClassVar + +from django_filters import rest_framework as filters +from utils import filters_mixins + +from .models import MLBadLumisection + + +class MLBadLumisectionFilter(filters_mixins.DatasetFilterMethods, filters_mixins.MEsMethods, filters.FilterSet): + class Meta: + model = MLBadLumisection + fields: ClassVar[dict[str, list[str]]] = { + "model_id": ["exact", "in"], + "dataset_id": ["exact"], + "me_id": ["exact"], + "run_number": ["exact"], + "ls_number": ["exact"], + } diff --git a/backend/ml_bad_lumisection/models.py b/backend/ml_bad_lumisection/models.py new file mode 100644 index 00000000..ed81abd6 --- /dev/null +++ b/backend/ml_bad_lumisection/models.py @@ -0,0 +1,31 @@ +from typing import ClassVar + +from django.db import models + + +class MLBadLumisection(models.Model): + """ + - Django doesn't support composite primary key + - The unique constraint set in this class do not exist in the database, + it is used here to select the composite primary key in the viewset and as a documentation + """ + + model_id = models.BigIntegerField(primary_key=True) + dataset_id = models.BigIntegerField() + file_id = models.BigIntegerField() + run_number = models.IntegerField() + ls_number = models.IntegerField() + me_id = models.IntegerField() + + class Meta: + managed = False + db_table = "fact_ml_bad_lumis" + constraints: ClassVar[list[models.Index]] = [ + models.UniqueConstraint( + name="fact_ml_bad_lumis_primary_key", + fields=["model_id", "dataset_id", "run_number", "ls_number", "me_id"], + ), + ] + + def __str__(self) -> str: + return f"MLBadLumisection <{self.me_id}@{self.ls_number}@{self.run_number}@{self.dataset_id}@{self.model_id}>" diff --git a/backend/ml_bad_lumisection/routers.py b/backend/ml_bad_lumisection/routers.py new file mode 100644 index 00000000..72287997 --- /dev/null +++ b/backend/ml_bad_lumisection/routers.py @@ -0,0 +1,7 @@ +from rest_framework import routers + +from .viewsets import MLBadLumisectionViewSet + + +router = routers.SimpleRouter() +router.register(r"ml-bad-lumisection", MLBadLumisectionViewSet, basename="ml-bad-lumisection") diff --git a/backend/ml_bad_lumisection/serializers.py b/backend/ml_bad_lumisection/serializers.py new file mode 100644 index 00000000..49387be1 --- /dev/null +++ b/backend/ml_bad_lumisection/serializers.py @@ -0,0 +1,9 @@ +from rest_framework import serializers + +from .models import MLBadLumisection + + +class MLBadLumisectionSerializer(serializers.ModelSerializer): + class Meta: + model = MLBadLumisection + fields = "__all__" diff --git a/backend/ml_bad_lumisection/viewsets.py b/backend/ml_bad_lumisection/viewsets.py new file mode 100644 index 00000000..861c1a2c --- /dev/null +++ b/backend/ml_bad_lumisection/viewsets.py @@ -0,0 +1,118 @@ +import logging +from typing import ClassVar + +from django.conf import settings +from django.shortcuts import get_object_or_404 +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page +from django.views.decorators.vary import vary_on_headers +from django_filters.rest_framework import DjangoFilterBackend +from ml_models_index.models import MLModelsIndex +from rest_framework import mixins, viewsets +from rest_framework.authentication import BaseAuthentication +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response +from utils.db_router import GenericViewSetRouter +from utils.rest_framework_cern_sso.authentication import ( + CERNKeycloakClientSecretAuthentication, + CERNKeycloakConfidentialAuthentication, +) + +from .filters import MLBadLumisectionFilter +from .models import MLBadLumisection +from .serializers import MLBadLumisectionSerializer + + +logger = logging.getLogger(__name__) +composite_pks = next(filter(lambda x: "primary_key" in x.name, MLBadLumisection._meta.constraints), None) + + +@method_decorator(cache_page(settings.CACHE_TTL), name="list") +@method_decorator(cache_page(settings.CACHE_TTL), name="get_object") +@method_decorator(vary_on_headers(settings.WORKSPACE_HEADER), name="list") +@method_decorator(vary_on_headers(settings.WORKSPACE_HEADER), name="get_object") +class MLBadLumisectionViewSet(GenericViewSetRouter, mixins.ListModelMixin, viewsets.GenericViewSet): + queryset = MLBadLumisection.objects.all().order_by(*composite_pks.fields) + serializer_class = MLBadLumisectionSerializer + filterset_class = MLBadLumisectionFilter + filter_backends: ClassVar[list[DjangoFilterBackend]] = [DjangoFilterBackend] + authentication_classes: ClassVar[list[BaseAuthentication]] = [ + CERNKeycloakClientSecretAuthentication, + CERNKeycloakConfidentialAuthentication, + ] + + @action( + detail=False, + methods=["GET"], + url_path=r"(?P\d+)/(?P\d+)/(?P\d+)/(?P\d+)/(?P\d+)", + ) + def get_object(self, request, model_id=None, dataset_id=None, run_number=None, ls_number=None, me_id=None): + # Since the MLBadLumisection table in the database has a composite primary key + # that Django doesn't support, we are defining this method + # as a custom retrieve method to query this table by the composite primary key + try: + model_id = int(model_id) + dataset_id = int(dataset_id) + run_number = int(run_number) + ls_number = int(ls_number) + me_id = int(me_id) + except ValueError as err: + raise ValidationError( + "model_id, dataset_id, run_number, ls_number and me_id must be valid integers." + ) from err + + queryset = self.get_queryset() + queryset = get_object_or_404( + queryset, model_id=model_id, dataset_id=dataset_id, run_number=run_number, ls_number=ls_number, me_id=me_id + ) + serializer = self.serializer_class(queryset) + return Response(serializer.data) + + @action(detail=False, methods=["GET"], url_path=r"cert-json") + def generate_certificate_json(self, request): + try: + dataset_id = int(request.query_params.get("dataset_id")) + run_number = list(map(int, request.query_params.get("run_number").split(","))) + model_ids = list(map(int, request.query_params.get("model_ids").split(","))) + except ValueError as err: + raise ValidationError( + "dataset_id and run_number must be valid integers and model_ids a valid list of integers" + ) from err + + # Select user's workspace + workspace = self.get_workspace() + + # Fetch models' metadata in the given workspace + models = MLModelsIndex.objects.using(workspace).filter(model_id__in=model_ids).all().values() + models = {qs.get("model_id"): qs for qs in models} + + # Fetch predictions for a given dataset, multiple runs from multiple models + queryset = self.get_queryset() + result = ( + queryset.filter(dataset_id=dataset_id, run_number__in=run_number, model_id__in=model_ids) + .all() + .order_by("run_number", "ls_number") + .values() + ) + result = [qs for qs in result] + + # Format certification json + response = {} + for run in run_number: + response[run] = {} + predictions_in_run = [res for res in result if res.get("run_number") == run] + unique_ls = [res.get("ls_number") for res in predictions_in_run] + for ls in unique_ls: + response[run][ls] = [] + predictions_in_ls = [res for res in predictions_in_run if res.get("ls_number") == ls] + for preds in predictions_in_ls: + model_id = preds.get("model_id") + me_id = preds.get("me_id") + filename = models[model_id].get("filename") + target_me = models[model_id].get("target_me") + response[run][ls].append( + {"model_id": model_id, "me_id": me_id, "filename": filename, "me": target_me} + ) + + return Response(response) From aae6c09c7ac4783f9f69e6d3cd3aa349fa96b2c1 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 10:55:58 +0200 Subject: [PATCH 11/30] refactor: move workspace selector logic to `get_workspace` function in db_router --- backend/utils/db_router.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/utils/db_router.py b/backend/utils/db_router.py index 93d20e59..31b3e969 100644 --- a/backend/utils/db_router.py +++ b/backend/utils/db_router.py @@ -16,16 +16,17 @@ def get_queryset(self): queryset = super().get_queryset() order_by = queryset.query.order_by queryset = queryset.model.objects - workspace = self.request.headers.get(settings.WORKSPACE_HEADER.capitalize()) + workspace = self.get_workspace() + queryset = queryset.using(workspace) + return queryset.all().order_by(*order_by) + def get_workspace(self): + workspace = self.request.headers.get(settings.WORKSPACE_HEADER.capitalize()) if workspace: if workspace not in settings.WORKSPACES.keys(): raise NotFound(detail=f"Workspace '{workspace}' not found", code=404) - queryset = queryset.using(workspace) else: user_roles = self.request.user.cern_roles workspace = get_workspace_from_role(user_roles) workspace = workspace or settings.DEFAULT_WORKSPACE - queryset = queryset.using(workspace) - - return queryset.all().order_by(*order_by) + return workspace From 68aea9a242953a4a755ba7737d1fdfa45da22378 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 11:06:21 +0200 Subject: [PATCH 12/30] refactor: change run_number and model_ids to run_number__in and model_id__in to keep consistent with other endpoints filters --- backend/ml_bad_lumisection/viewsets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/ml_bad_lumisection/viewsets.py b/backend/ml_bad_lumisection/viewsets.py index 861c1a2c..f7c8f991 100644 --- a/backend/ml_bad_lumisection/viewsets.py +++ b/backend/ml_bad_lumisection/viewsets.py @@ -73,8 +73,8 @@ def get_object(self, request, model_id=None, dataset_id=None, run_number=None, l def generate_certificate_json(self, request): try: dataset_id = int(request.query_params.get("dataset_id")) - run_number = list(map(int, request.query_params.get("run_number").split(","))) - model_ids = list(map(int, request.query_params.get("model_ids").split(","))) + run_number = list(map(int, request.query_params.get("run_number__in").split(","))) + model_id = list(map(int, request.query_params.get("model_id__in").split(","))) except ValueError as err: raise ValidationError( "dataset_id and run_number must be valid integers and model_ids a valid list of integers" @@ -84,13 +84,13 @@ def generate_certificate_json(self, request): workspace = self.get_workspace() # Fetch models' metadata in the given workspace - models = MLModelsIndex.objects.using(workspace).filter(model_id__in=model_ids).all().values() + models = MLModelsIndex.objects.using(workspace).filter(model_id__in=model_id).all().values() models = {qs.get("model_id"): qs for qs in models} # Fetch predictions for a given dataset, multiple runs from multiple models queryset = self.get_queryset() result = ( - queryset.filter(dataset_id=dataset_id, run_number__in=run_number, model_id__in=model_ids) + queryset.filter(dataset_id=dataset_id, run_number__in=run_number, model_id__in=model_id) .all() .order_by("run_number", "ls_number") .values() From 4ba00336f69b3eecadc6426507b17078ec1a55f5 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Thu, 27 Jun 2024 13:30:17 +0200 Subject: [PATCH 13/30] refactor: add new endpoints to swagger config --- backend/static/swagger.json | 623 +++++++++++++++++++++++++++++++++++- 1 file changed, 621 insertions(+), 2 deletions(-) diff --git a/backend/static/swagger.json b/backend/static/swagger.json index 8ad9cdeb..5fb120d3 100644 --- a/backend/static/swagger.json +++ b/backend/static/swagger.json @@ -1931,7 +1931,506 @@ {} ] } - } + }, + "/api/v1/ml-models-index/": { + "get": { + "operationId": "listMLModelsIndex", + "description": "", + "parameters": [ + { + "name": "next_token", + "required": false, + "in": "query", + "description": "next_token", + "schema": { + "type": "string" + } + }, + { + "name": "model_id", + "required": false, + "in": "query", + "description": "model_id", + "schema": { + "type": "integer" + } + }, + { + "name": "model_id__in", + "required": false, + "in": "query", + "description": "model_id__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "filename", + "required": false, + "in": "query", + "description": "filename", + "schema": { + "type": "string" + } + }, + { + "name": "filename__regex", + "required": false, + "in": "query", + "description": "filename__regex", + "schema": { + "type": "string" + } + }, + { + "name": "target_me", + "required": false, + "in": "query", + "description": "target_me", + "schema": { + "type": "string" + } + }, + { + "name": "target_me__regex", + "required": false, + "in": "query", + "description": "target_me__regex", + "schema": { + "type": "string" + } + }, + { + "name": "active", + "required": false, + "in": "query", + "description": "active", + "schema": { + "type": "boolean" + } + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "next": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?next_token=cD0zMz" + }, + "previous": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?next_token=cj0xJnA" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MLModelsIndex" + } + } + } + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Models Index" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, + "/api/v1/ml-models-index/{model_id}/": { + "get": { + "operationId": "retrieveMLModelsIndex", + "description": "", + "parameters": [ + { + "name": "model_id", + "in": "path", + "required": true, + "description": "A unique value identifying this ml model in the index.", + "schema": { + "type": "string" + } + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MLModelsIndex" + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Models Index" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, + "/api/v1/ml-bad-lumisection/": { + "get": { + "operationId": "listMLBadLumisection", + "description": "", + "parameters": [ + { + "name": "next_token", + "required": false, + "in": "query", + "description": "next_token", + "schema": { + "type": "string" + } + }, + { + "name": "model_id", + "required": false, + "in": "query", + "description": "model_id", + "schema": { + "type": "integer" + } + }, + { + "name": "model_id__in", + "required": false, + "in": "query", + "description": "model_id__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "dataset", + "required": false, + "in": "query", + "description": "dataset", + "schema": { + "type": "string" + } + }, + { + "name": "dataset__regex", + "required": false, + "in": "query", + "description": "dataset__regex", + "schema": { + "type": "string" + } + }, + { + "name": "me", + "required": false, + "in": "query", + "description": "me", + "schema": { + "type": "string" + } + }, + { + "name": "me__regex", + "required": false, + "in": "query", + "description": "me__regex", + "schema": { + "type": "string" + } + }, + { + "name": "run_number", + "required": false, + "in": "query", + "description": "run_number", + "schema": { + "type": "integer" + } + }, + { + "name": "ls_number", + "required": false, + "in": "query", + "description": "ls_number", + "schema": { + "type": "string" + } + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "next": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?next_token=cD0zMz" + }, + "previous": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?next_token=cj0xJnA" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MLBadLumisection" + } + } + } + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Bad Lumisection" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, + "/api/v1/ml-bad-lumisection/{model_id}/{dataset_id}/{run_number}/{ls_number}/{me_id}/": { + "get": { + "operationId": "retrieveMLBadLumisection", + "description": "", + "parameters": [ + { + "name": "model_id", + "in": "path", + "required": true, + "description": "A unique value identifying this ml model.", + "schema": { + "type": "string" + } + }, + { + "name": "dataset_id", + "in": "path", + "required": true, + "description": "A unique value identifying the dataset.", + "schema": { + "type": "string" + } + }, + { + "name": "run_number", + "in": "path", + "required": true, + "description": "A unique value identifying the run.", + "schema": { + "type": "string" + } + }, + { + "name": "ls_number", + "in": "path", + "required": true, + "description": "A unique value identifying the lumisection.", + "schema": { + "type": "string" + } + }, + { + "name": "me_id", + "in": "path", + "required": true, + "description": "A unique value identifying the monitoring element.", + "schema": { + "type": "string" + } + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MLBadLumisection" + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Bad Lumisection" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, + "/api/v1/ml-bad-lumisection/cert-json": { + "get": { + "operationId": "certJsonMLBadLumisection", + "description": "", + "parameters": [ + { + "name": "model_id__in", + "required": false, + "in": "query", + "description": "model_id__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "dataset_id", + "required": false, + "in": "query", + "description": "dataset_id", + "schema": { + "type": "integer" + } + }, + { + "name": "run_number__in", + "required": false, + "in": "query", + "description": "run_number__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MLBadLumisectionCertJson" + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Bad Lumisection" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, }, "components": { "schemas": { @@ -2415,7 +2914,127 @@ "entries", "data" ] - } + }, + "MLModelsIndex": { + "type": "object", + "properties": { + "model_id": { + "type": "integer", + "maximum": 9223372036854776000, + "minimum": -9223372036854776000, + "format": "int64" + }, + "filename": { + "type": "string", + "maxLength": 255 + }, + "target_me": { + "type": "string", + "maxLength": 255 + }, + "thr": { + "type": "number", + }, + "active": { + "type": "boolean", + } + }, + "required": [ + "model_id", + "filename", + "target_me", + "thr", + "active" + ] + }, + "MLBadLumisection": { + "type": "object", + "properties": { + "model_id": { + "type": "integer", + "maximum": 9223372036854776000, + "minimum": -9223372036854776000, + "format": "int64" + }, + "dataset_id": { + "type": "integer", + "maximum": 9223372036854776000, + "minimum": -9223372036854776000, + "format": "int64" + }, + "file_id": { + "type": "integer", + "maximum": 9223372036854776000, + "minimum": -9223372036854776000, + "format": "int64" + }, + "run_number": { + "type": "integer", + "maximum": 2147483647, + "minimum": -2147483648 + }, + "ls_number": { + "type": "integer", + "maximum": 2147483647, + "minimum": -2147483648 + }, + "me_id": { + "type": "integer", + "maximum": 2147483647, + "minimum": -2147483648 + } + }, + "required": [ + "model_id", + "dataset_id", + "file_id", + "run_number", + "ls_number", + "me_id" + ] + }, + "MLBadLumisectionCertJson": { + "type": "object", + "properties": { + "run_number": { + "type": "object", + "properties": { + "ls_number": { + "type": "array", + "items": { + "type": "object", + "properties": { + "model_id": { + "type": "integer", + "maximum": 2147483647, + "minimum": -2147483648, + }, + "me_id": { + "type": "integer", + "maximum": 2147483647, + "minimum": -2147483648, + }, + "filename": { + "type": "string" + }, + "me": { + "type": "string" + }, + } + } + } + } + } + }, + "required": [ + "run_number", + "ls_number", + "model_id", + "me_id", + "filename", + "me" + ] + }, }, "securitySchemes": { "Client Secret Key": { From c29de9a397f7f8c89122b987630d8f8dde2b1b9b Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Mon, 1 Jul 2024 16:03:36 +0200 Subject: [PATCH 14/30] chore: add common lib --- backend/utils/common.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 backend/utils/common.py diff --git a/backend/utils/common.py b/backend/utils/common.py new file mode 100644 index 00000000..48815ba7 --- /dev/null +++ b/backend/utils/common.py @@ -0,0 +1,7 @@ +import itertools + + +def list_to_range(i): + for _, b in itertools.groupby(enumerate(i), lambda pair: pair[1] - pair[0]): + b = list(b) + yield b[0][1], b[-1][1] From dfc33f6d2ea7acf02585971b3be4dbde6974f9c6 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Mon, 1 Jul 2024 16:04:09 +0200 Subject: [PATCH 15/30] feat: generate golden-json like response by filtering bad lumisection from all ingested lumisections --- backend/ml_bad_lumisection/viewsets.py | 51 +++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/backend/ml_bad_lumisection/viewsets.py b/backend/ml_bad_lumisection/viewsets.py index f7c8f991..3532dcf4 100644 --- a/backend/ml_bad_lumisection/viewsets.py +++ b/backend/ml_bad_lumisection/viewsets.py @@ -7,12 +7,14 @@ from django.views.decorators.cache import cache_page from django.views.decorators.vary import vary_on_headers from django_filters.rest_framework import DjangoFilterBackend +from lumisection.models import Lumisection from ml_models_index.models import MLModelsIndex from rest_framework import mixins, viewsets from rest_framework.authentication import BaseAuthentication from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.response import Response +from utils.common import list_to_range from utils.db_router import GenericViewSetRouter from utils.rest_framework_cern_sso.authentication import ( CERNKeycloakClientSecretAuthentication, @@ -97,7 +99,7 @@ def generate_certificate_json(self, request): ) result = [qs for qs in result] - # Format certification json + # Format bad lumi certification json response = {} for run in run_number: response[run] = {} @@ -116,3 +118,50 @@ def generate_certificate_json(self, request): ) return Response(response) + + @action(detail=False, methods=["GET"], url_path=r"golden-json") + def generate_golden_json(self, request): + try: + dataset_id = int(request.query_params.get("dataset_id")) + run_number = list(map(int, request.query_params.get("run_number__in").split(","))) + model_id = list(map(int, request.query_params.get("model_id__in").split(","))) + except ValueError as err: + raise ValidationError( + "dataset_id and run_number must be valid integers and model_ids a valid list of integers" + ) from err + + # Select user's workspace + workspace = self.get_workspace() + + # Fetch predictions for a given dataset, multiple runs from multiple models + queryset = self.get_queryset() + result = ( + queryset.filter(dataset_id=dataset_id, run_number__in=run_number, model_id__in=model_id) + .all() + .order_by("run_number", "ls_number") + .values() + ) + result = [qs for qs in result] + + # Generate ML golden json + response = {} + for run in run_number: + queryset = self.get_queryset() + bad_lumis = ( + queryset.filter(dataset_id=dataset_id, run_number=run, model_id__in=model_id) + .all() + .order_by("ls_number") + .values_list("ls_number", flat=True) + .distinct() + ) + bad_lumis = [qs for qs in bad_lumis] + all_lumis = ( + Lumisection.objects.using(workspace) + .filter(dataset_id=dataset_id, run_number=run) + .all() + .values_list("ls_number", flat=True) + ) + good_lumis = [ls for ls in all_lumis if ls not in bad_lumis] + response[run] = list_to_range(good_lumis) + + return Response(response) From 4d4a8ecbd32caa6cbeb70f9dce13fb956fad32e4 Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Mon, 1 Jul 2024 17:59:09 +0200 Subject: [PATCH 16/30] chore: add ml-bad-lumisection golden-json endpoint to swagger conf --- backend/static/swagger.json | 101 ++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/backend/static/swagger.json b/backend/static/swagger.json index 5fb120d3..3daa42c8 100644 --- a/backend/static/swagger.json +++ b/backend/static/swagger.json @@ -2431,6 +2431,84 @@ ] } }, + "/api/v1/ml-bad-lumisection/golden-json": { + "get": { + "operationId": "goldenJsonMLBadLumisection", + "description": "", + "parameters": [ + { + "name": "model_id__in", + "required": false, + "in": "query", + "description": "model_id__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "dataset_id", + "required": false, + "in": "query", + "description": "dataset_id", + "schema": { + "type": "integer" + } + }, + { + "name": "run_number__in", + "required": false, + "in": "query", + "description": "run_number__in", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + }, + "style": "form", + "explode": false + }, + { + "name": "workspace", + "required": false, + "in": "header", + "description": "workspace", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MLBadLumisectionGoldenJson" + } + } + }, + "description": "" + } + }, + "tags": [ + "ML Bad Lumisection" + ], + "security": [ + { + "Client Secret Key": [] + }, + { + "Confidential JWT Token": [] + }, + {} + ] + } + }, }, "components": { "schemas": { @@ -3035,6 +3113,29 @@ "me" ] }, + "MLBadLumisectionGoldenJson": { + "type": "object", + "properties": { + "run_number": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + }, + "example": [128, 145] + } + } + }, + "required": [ + "run_number", + "ls_number", + "model_id", + "me_id", + "filename", + "me" + ] + }, }, "securitySchemes": { "Client Secret Key": { From 59e8528f90ceb5f4578c6af1ffb3adbae49ef48b Mon Sep 17 00:00:00 2001 From: Gabriel Moreira Date: Tue, 2 Jul 2024 22:41:14 +0200 Subject: [PATCH 17/30] feat[wip]: add predictions route to display bad lumisections from selected models on a given dataset and multiple runs --- frontend/src/components/navbar.jsx | 5 + frontend/src/components/routes.jsx | 4 + frontend/src/views/index.jsx | 4 + frontend/src/views/machineLearning/index.js | 3 + .../src/views/machineLearning/predictions.jsx | 150 ++++++++ package.json | 1 + yarn.lock | 325 +++++++++++++++++- 7 files changed, 485 insertions(+), 7 deletions(-) create mode 100644 frontend/src/views/machineLearning/index.js create mode 100644 frontend/src/views/machineLearning/predictions.jsx diff --git a/frontend/src/components/navbar.jsx b/frontend/src/components/navbar.jsx index 3be18359..e69a868e 100644 --- a/frontend/src/components/navbar.jsx +++ b/frontend/src/components/navbar.jsx @@ -69,6 +69,11 @@ const AppNavbar = ({ Lumisections + + + Predictions + +