Enhance local mode support for LLM inference without model data

src/sagemaker/local/entities.py

@@ -615,6 +615,10 @@ def serve(self):
         self.container = _SageMakerContainer(
             instance_type, instance_count, image, self.local_session
         )
+
+        if "ModelDataUrl" not in  self.primary_container.keys():
+            self.primary_container["ModelDataUrl"] = None
+
         self.container.serve(
             self.primary_container["ModelDataUrl"], self.primary_container["Environment"]
         )

src/sagemaker/local/image.py

@@ -340,7 +340,16 @@ def serve(self, model_dir, environment):
         self.container_root = self._create_tmp_folder()
         logger.info("creating hosting dir in %s", self.container_root)
 
-        volumes = self._prepare_serving_volumes(model_dir)
+        if model_dir is not None: 
+            volumes = self._prepare_serving_volumes(model_dir)
+        else:
+            volumes = None
+            # if model_data is None, then force create ../sagemaker-local under 
+            # contianer root
+            os.makedirs(
+                os.path.join(self.container_root, self.hosts[0]), 
+                exist_ok=True
+            )
 
         # If the user script was passed as a file:// mount it to the container.
         if sagemaker.estimator.DIR_PARAM_NAME.upper() in environment:
@@ -869,7 +878,7 @@ def _create_docker_host(
         if self.instance_type == "local_gpu":
             host_config["deploy"] = {
                 "resources": {
-                    "reservations": {"devices": [{"count": "all", "capabilities": ["gpu"]}]}
+                    "reservations": {"devices": [{"driver": "nvidia", "count": "all", "capabilities": ["gpu"]}]}
                 }
             }
 

tests/integ/sagemaker/local_mode/sample_inference_script.py

@@ -0,0 +1,30 @@
+import os
+
+# Install the required package
+os.system("pip install transformers==4.18.0")
+
+from transformers import pipeline
+
+CSV_CONTENT_TYPE = 'text/csv'
+
+
+def model_fn(model_dir):
+    sentiment_analysis = pipeline(
+        "sentiment-analysis",
+        model=model_dir,
+        tokenizer=model_dir,
+        return_all_scores=True
+    )
+    return sentiment_analysis
+
+
+def input_fn(serialized_input_data, content_type=CSV_CONTENT_TYPE):
+    if content_type == CSV_CONTENT_TYPE:
+        input_data = serialized_input_data.splitlines()
+        return input_data
+    else:
+        raise Exception('Requested unsupported ContentType in Accept: ' + content_type)
+
+
+def predict_fn(input_data, model):
+    return model(input_data)

tests/integ/sagemaker/local_mode/sample_processing_script.py

@@ -0,0 +1,41 @@
+import pandas as pd
+import numpy as np
+import argparse
+import os
+from sklearn.preprocessing import OrdinalEncoder
+
+def _parse_args():
+
+    parser = argparse.ArgumentParser()
+
+    # Data, model, and output directories
+    # model_dir is always passed in from SageMaker. By default this is a S3 path under the default bucket.
+    parser.add_argument('--filepath', type=str, default='/opt/ml/processing/input/')
+    parser.add_argument('--filename', type=str, default='bank-additional-full.csv')
+    parser.add_argument('--outputpath', type=str, default='/opt/ml/processing/output/')
+    parser.add_argument('--categorical_features', type=str, default='y, job, marital, education, default, housing, loan, contact, month, day_of_week, poutcome')
+
+    return parser.parse_known_args()
+
+if __name__=="__main__":
+    # Process arguments
+    args, _ = _parse_args()
+    # Load data
+    df = pd.read_csv(os.path.join(args.filepath, args.filename))
+    # Change the value . into _
+    df = df.replace(regex=r'\.', value='_')
+    df = df.replace(regex=r'\_$', value='')
+    # Add two new indicators
+    df["no_previous_contact"] = (df["pdays"] == 999).astype(int)
+    df["not_working"] = df["job"].isin(["student", "retired", "unemployed"]).astype(int)
+    df = df.drop(['duration', 'emp.var.rate', 'cons.price.idx', 'cons.conf.idx', 'euribor3m', 'nr.employed'], axis=1)
+    # Encode the categorical features
+    df = pd.get_dummies(df)
+    # Train, test, validation split
+    train_data, validation_data, test_data = np.split(df.sample(frac=1, random_state=42), [int(0.7 * len(df)), int(0.9 * len(df))])   # Randomly sort the data then split out first 70%, second 20%, and last 10%
+    # Local store
+    pd.concat([train_data['y_yes'], train_data.drop(['y_yes','y_no'], axis=1)], axis=1).to_csv(os.path.join(args.outputpath, 'train/train.csv'), index=False, header=False)
+    pd.concat([validation_data['y_yes'], validation_data.drop(['y_yes','y_no'], axis=1)], axis=1).to_csv(os.path.join(args.outputpath, 'validation/validation.csv'), index=False, header=False)
+    test_data['y_yes'].to_csv(os.path.join(args.outputpath, 'test/test_y.csv'), index=False, header=False)
+    test_data.drop(['y_yes','y_no'], axis=1).to_csv(os.path.join(args.outputpath, 'test/test_x.csv'), index=False, header=False)
+    print("## Processing complete. Exiting.")

tests/integ/sagemaker/local_mode/sample_training_script.py

@@ -0,0 +1,88 @@
+#  Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+from __future__ import print_function
+
+import argparse
+import os
+
+import joblib
+import pandas as pd
+from sklearn import tree
+from sklearn.metrics import mean_squared_error
+
+if __name__ == "__main__":
+    print("Training Started")
+    parser = argparse.ArgumentParser()
+
+    # Hyperparameters are described here. In this simple example we are just including one hyperparameter.
+    parser.add_argument("--max_leaf_nodes", type=int, default=-1)
+
+    # Sagemaker specific arguments. Defaults are set in the environment variables.
+    parser.add_argument("--output-data-dir", type=str, default=os.environ["SM_OUTPUT_DATA_DIR"])
+    parser.add_argument("--model-dir", type=str, default=os.environ["SM_MODEL_DIR"])
+    parser.add_argument("--train", type=str, default=os.environ["SM_CHANNEL_TRAIN"])
+    parser.add_argument("--validation", type=str, default=os.environ["SM_CHANNEL_VALIDATION"])
+
+    args = parser.parse_args()
+    print("Got Args: {}".format(args))
+
+    # Take the set of files and read them all into a single pandas dataframe
+    input_files = [os.path.join(args.train, file) for file in os.listdir(args.train)]
+    if len(input_files) == 0:
+        raise ValueError(
+            (
+                "There are no files in {}.\n"
+                + "This usually indicates that the channel ({}) was incorrectly specified,\n"
+                + "the data specification in S3 was incorrectly specified or the role specified\n"
+                + "does not have permission to access the data."
+            ).format(args.train, "train")
+        )
+    raw_data = [pd.read_csv(file, header=None, engine="python") for file in input_files]
+    train_data = pd.concat(raw_data)
+
+    # labels are in the first column
+    train_y = train_data.iloc[:, 0]
+    train_X = train_data.iloc[:, 1:]
+
+    # Here we support a single hyperparameter, 'max_leaf_nodes'. Note that you can add as many
+    # as your training my require in the ArgumentParser above.
+    max_leaf_nodes = args.max_leaf_nodes
+
+    # Now use scikit-learn's decision tree regression to train the model.
+    clf = tree.DecisionTreeRegressor(max_leaf_nodes=max_leaf_nodes)
+    clf = clf.fit(train_X, train_y)
+
+    input_files = [os.path.join(args.validation, file) for file in os.listdir(args.validation)]
+    raw_data = [pd.read_csv(file, header=None, engine="python") for file in input_files]
+    validation_data = pd.concat(raw_data)
+    # labels are in the first column
+    validation_y = validation_data.iloc[:, 0]
+    validation_X = validation_data.iloc[:, 1:]
+    #
+    predictions = clf.predict(validation_X)
+    error = mean_squared_error(predictions, validation_y)
+    print(f"RMSE: {error}")
+    # Print the coefficients of the trained classifier, and save the coefficients
+    joblib.dump(clf, os.path.join(args.model_dir, "model.joblib"))
+
+    print("Training Completed")
+
+
+def model_fn(model_dir):
+    """Deserialized and return fitted model
+
+    Note that this should have the same name as the serialized model in the main method
+    """
+    clf = joblib.load(os.path.join(model_dir, "model.joblib"))
+    return clf