From 40fce28cba5fdfd01905ba3d96382d5b10ba8ec0 Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Tue, 22 Nov 2022 10:49:41 +0530 Subject: [PATCH 1/6] Trigger build Signed-off-by: Shrinath Suresh From 650d3ad1d5234e47dc93b387e651e771f1aad138 Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Wed, 7 Dec 2022 08:17:13 +0530 Subject: [PATCH 2/6] MLflow 2.0 fixes Signed-off-by: Shrinath Suresh --- examples/BertNewsClassification/README.md | 7 +++++++ examples/IrisClassification/predict.py | 3 ++- examples/MNIST/mnist_model.py | 15 ++++++++------- examples/TextClassification/predict.py | 2 +- mlflow_torchserve/__init__.py | 23 +++++++++++++++++++---- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/examples/BertNewsClassification/README.md b/examples/BertNewsClassification/README.md index 67d647ad..863cb989 100644 --- a/examples/BertNewsClassification/README.md +++ b/examples/BertNewsClassification/README.md @@ -82,6 +82,13 @@ python news_classifier.py \ --model_save_path /home/ubuntu/mlflow-torchserve/examples/BertNewsClassification/models ``` +To run the training script in GPU environment: +``` +torchrun news_classifier.py \ + --max_epochs 5 \ + --model_save_path /home/ubuntu/mlflow-torchserve/examples/BertNewsClassification/models +``` + ## Starting TorchServe create an empty directory `model_store` and run the following command to start torchserve. diff --git a/examples/IrisClassification/predict.py b/examples/IrisClassification/predict.py index 9beb3af0..0ed82411 100644 --- a/examples/IrisClassification/predict.py +++ b/examples/IrisClassification/predict.py @@ -30,7 +30,8 @@ def predict(parser_args): df[column] = df[column].astype("double") prediction = plugin.predict(deployment_name=parser_args["deployment_name"], df=input_data) - print("Prediction Result {}".format(prediction)) + + print("Prediction Result {}".format(prediction.to_json())) if __name__ == "__main__": diff --git a/examples/MNIST/mnist_model.py b/examples/MNIST/mnist_model.py index 5c5b8f51..8cf2921a 100644 --- a/examples/MNIST/mnist_model.py +++ b/examples/MNIST/mnist_model.py @@ -279,8 +279,6 @@ def get_model(trainer): if dict_args[argument] == "None": dict_args[argument] = None - mlflow.pytorch.autolog() - model = LightningMNISTClassifier(**dict_args) dm = MNISTDataModule(**dict_args) @@ -289,11 +287,14 @@ def get_model(trainer): trainer = pl.Trainer.from_argparse_args(args) - trainer.fit(model, dm) - trainer.test(datamodule=dm) - - run = mlflow.active_run() + mlflow.pytorch.autolog() + with mlflow.start_run() as run: + trainer.fit(model, dm) + trainer.test(datamodule=dm) + active_run = mlflow.active_run() if dict_args["register"] == "true": - mlflow.register_model(model_uri=run.info.artifact_uri, name=dict_args["registration_name"]) + mlflow.register_model( + model_uri=active_run.info.artifact_uri, name=dict_args["registration_name"] + ) else: torch.save(trainer.lightning_module.state_dict(), "model.pth") diff --git a/examples/TextClassification/predict.py b/examples/TextClassification/predict.py index 03da7872..6a5488c3 100644 --- a/examples/TextClassification/predict.py +++ b/examples/TextClassification/predict.py @@ -10,7 +10,7 @@ def predict(parser_args): text = fp.read() plugin = get_deploy_client(parser_args["target"]) prediction = plugin.predict(parser_args["deployment_name"], json.dumps(text)) - print("Prediction Result {}".format(prediction)) + print("Prediction Result {}".format(prediction.to_json())) if __name__ == "__main__": diff --git a/mlflow_torchserve/__init__.py b/mlflow_torchserve/__init__.py index fb2ce2ec..8672943c 100644 --- a/mlflow_torchserve/__init__.py +++ b/mlflow_torchserve/__init__.py @@ -12,8 +12,10 @@ from mlflow_torchserve.config import Config from mlflow.deployments import BaseDeploymentClient, get_deploy_client +from mlflow.deployments import PredictionsResponse from mlflow.tracking.artifact_utils import _download_artifact_from_uri from mlflow.models.model import Model +from mlflow.utils.proto_json_utils import NumpyEncoder, _get_jsonable_obj _logger = logging.getLogger(__name__) @@ -21,6 +23,21 @@ _DEFAULT_TORCHSERVE_LOCAL_MANAGEMENT_PORT = "8081" +class CustomPredictionsResponse(PredictionsResponse): + def __init__(self, resp): + super(CustomPredictionsResponse, self).__init__(self) + self.resp = resp + + def to_json(self, path=None): + if path is not None: + with open(path, "w") as f: + json.dump(dict(self), f) + elif self.resp is not None: + return self.resp + else: + return json.dumps(dict(self)) + + class TorchServePlugin(BaseDeploymentClient): def __init__(self, uri): @@ -268,7 +285,6 @@ def predict(self, deployment_name, df): :return: output - Returns the predicted value """ - url = "{api}/{predictions}/{name}".format( api=self.inference_api, predictions="predictions", name=deployment_name ) @@ -286,14 +302,14 @@ def predict(self, deployment_name, df): raise ValueError("Unable to parse input json string: {}".format(e)) resp = requests.post(url, data) + cust_resp = CustomPredictionsResponse(resp.text) if resp.status_code != 200: raise Exception( "Unable to infer the results for the name %s. " "Server returned status code %s and response: %s" % (deployment_name, resp.status_code, resp.content) ) - - return resp.text + return cust_resp def explain(self, deployment_name, df): """ @@ -330,7 +346,6 @@ def explain(self, deployment_name, df): "Server returned status code %s and response: %s" % (deployment_name, resp.status_code, resp.content) ) - return resp.text def __generate_mar_file( From ac56dcfdbcc4800d4b41d43e449a1643a58f5253 Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Wed, 7 Dec 2022 10:16:16 +0530 Subject: [PATCH 3/6] Accuracy fix Signed-off-by: Shrinath Suresh --- examples/IrisClassification/iris_classification.py | 6 +++--- examples/MNIST/mnist_model.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/IrisClassification/iris_classification.py b/examples/IrisClassification/iris_classification.py index 2605c8bc..782ad95a 100644 --- a/examples/IrisClassification/iris_classification.py +++ b/examples/IrisClassification/iris_classification.py @@ -21,9 +21,9 @@ class IrisClassification(pl.LightningModule): def __init__(self, **kwargs): super(IrisClassification, self).__init__() - self.train_acc = Accuracy() - self.val_acc = Accuracy() - self.test_acc = Accuracy() + self.train_acc = Accuracy(task="multiclass") + self.val_acc = Accuracy(task="multiclass") + self.test_acc = Accuracy(task="multiclass") self.args = kwargs self.fc1 = nn.Linear(4, 10) diff --git a/examples/MNIST/mnist_model.py b/examples/MNIST/mnist_model.py index 8cf2921a..80d9000e 100644 --- a/examples/MNIST/mnist_model.py +++ b/examples/MNIST/mnist_model.py @@ -120,9 +120,9 @@ def __init__(self, **kwargs): """ super(LightningMNISTClassifier, self).__init__() - self.train_acc = Accuracy() - self.val_acc = Accuracy() - self.test_acc = Accuracy() + self.train_acc = Accuracy(task="multiclass") + self.val_acc = Accuracy(task="multiclass") + self.test_acc = Accuracy(task="multiclass") # mnist images are (1, 28, 28) (channels, width, height) self.layer_1 = torch.nn.Linear(28 * 28, 128) From 9f7517f80923d4b9153e9217b4a326c7dda73b67 Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Thu, 15 Dec 2022 07:54:09 +0530 Subject: [PATCH 4/6] UT fixes Signed-off-by: Shrinath Suresh --- examples/IrisClassification/iris_classification.py | 6 +++--- examples/MNIST/mnist_model.py | 6 +++--- mlflow_torchserve/__init__.py | 6 ++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/examples/IrisClassification/iris_classification.py b/examples/IrisClassification/iris_classification.py index 782ad95a..a5933a93 100644 --- a/examples/IrisClassification/iris_classification.py +++ b/examples/IrisClassification/iris_classification.py @@ -21,9 +21,9 @@ class IrisClassification(pl.LightningModule): def __init__(self, **kwargs): super(IrisClassification, self).__init__() - self.train_acc = Accuracy(task="multiclass") - self.val_acc = Accuracy(task="multiclass") - self.test_acc = Accuracy(task="multiclass") + self.train_acc = Accuracy(task="multiclass", num_classes=3) + self.val_acc = Accuracy(task="multiclass", num_classes=3) + self.test_acc = Accuracy(task="multiclass", num_classes=3) self.args = kwargs self.fc1 = nn.Linear(4, 10) diff --git a/examples/MNIST/mnist_model.py b/examples/MNIST/mnist_model.py index 80d9000e..d6d66697 100644 --- a/examples/MNIST/mnist_model.py +++ b/examples/MNIST/mnist_model.py @@ -120,9 +120,9 @@ def __init__(self, **kwargs): """ super(LightningMNISTClassifier, self).__init__() - self.train_acc = Accuracy(task="multiclass") - self.val_acc = Accuracy(task="multiclass") - self.test_acc = Accuracy(task="multiclass") + self.train_acc = Accuracy(task="multiclass", num_classes=10) + self.val_acc = Accuracy(task="multiclass", num_classes=10) + self.test_acc = Accuracy(task="multiclass", num_classes=10) # mnist images are (1, 28, 28) (channels, width, height) self.layer_1 = torch.nn.Linear(28 * 28, 128) diff --git a/mlflow_torchserve/__init__.py b/mlflow_torchserve/__init__.py index 8672943c..cf357dc7 100644 --- a/mlflow_torchserve/__init__.py +++ b/mlflow_torchserve/__init__.py @@ -31,11 +31,9 @@ def __init__(self, resp): def to_json(self, path=None): if path is not None: with open(path, "w") as f: - json.dump(dict(self), f) - elif self.resp is not None: - return self.resp + json.dump(self.resp, f) else: - return json.dumps(dict(self)) + return self.resp class TorchServePlugin(BaseDeploymentClient): From 45a17fe2471a1f9eb0d5e91156e611a64877cd51 Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Thu, 15 Dec 2022 07:55:23 +0530 Subject: [PATCH 5/6] Optimizing imports Signed-off-by: Shrinath Suresh --- mlflow_torchserve/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mlflow_torchserve/__init__.py b/mlflow_torchserve/__init__.py index cf357dc7..81894c42 100644 --- a/mlflow_torchserve/__init__.py +++ b/mlflow_torchserve/__init__.py @@ -1,21 +1,20 @@ -import docker import json -import subprocess -import time import logging import os -import pandas as pd -import torch +import subprocess +import time from pathlib import Path +import docker +import pandas as pd import requests -from mlflow_torchserve.config import Config - +import torch from mlflow.deployments import BaseDeploymentClient, get_deploy_client from mlflow.deployments import PredictionsResponse -from mlflow.tracking.artifact_utils import _download_artifact_from_uri from mlflow.models.model import Model -from mlflow.utils.proto_json_utils import NumpyEncoder, _get_jsonable_obj +from mlflow.tracking.artifact_utils import _download_artifact_from_uri + +from mlflow_torchserve.config import Config _logger = logging.getLogger(__name__) From a0d445080dd6a25641ded67476286e242816571f Mon Sep 17 00:00:00 2001 From: Shrinath Suresh Date: Thu, 15 Dec 2022 08:37:46 +0530 Subject: [PATCH 6/6] MNIST UT fix Signed-off-by: Shrinath Suresh --- examples/MNIST/predict.py | 2 +- examples/Titanic/README.md | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/MNIST/predict.py b/examples/MNIST/predict.py index 190d9e35..6c44a45c 100644 --- a/examples/MNIST/predict.py +++ b/examples/MNIST/predict.py @@ -15,7 +15,7 @@ def predict(parser_args): image_tensor = mnist_transforms(img) prediction = plugin.predict(parser_args["deployment_name"], image_tensor) - print("Prediction Result {}".format(prediction)) + print("Prediction Result {}".format(prediction.to_json())) if __name__ == "__main__": diff --git a/examples/Titanic/README.md b/examples/Titanic/README.md index 2791ca69..dc957d8e 100644 --- a/examples/Titanic/README.md +++ b/examples/Titanic/README.md @@ -48,7 +48,7 @@ mlflow run . --no-conda * Step - 3: Create an empty directory model_store and run the following command to start torchserve. ```bash - torchserve --start --model-store model_store/ --ts-config config.properties + torchserve --start --model-store model_store/ ``` ## Creating a new deployment @@ -67,13 +67,17 @@ For testing, we are going to use a sample test record placed in test_data folder Run the following command to invoke prediction on test record, whose output is stored in output.json file. -`mlflow deployments predict --name titanic --target torchserve --input-path test_data/input.json --output-path output.json` +``` +mlflow deployments predict --name titanic --target torchserve --input-path test_data/input.json --output-path output.json +``` This model will classify the test record as survived or not survived and store it in `output.json` Run the below command to invoke explain for feature importance attributions on test record. It will save the attribution image attributions_imp.png in test_data folder. -` mlflow deployments explain -t torchserve --name titanic --input-path test_data/input.json` +``` +mlflow deployments explain -t torchserve --name titanic --input-path test_data/input.json +``` this explanations command give us the average attribution for each feature. From the feature attribution information, we obtain some interesting insights regarding the importance of various features.