Move local endpoints from torchscript to custom fastapi server

Author: stepjam

.gitattributes

@@ -1,4 +1,4 @@
-*.mar filter=lfs diff=lfs merge=lfs -text
+*.nc.zip filter=lfs diff=lfs merge=lfs -text
 *.stl filter=lfs diff=lfs merge=lfs -text
 *.obj filter=lfs diff=lfs merge=lfs -text
 *.dae filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -108,7 +108,7 @@ ehthumbs.db
 MUJOCO_LOG.TXT
 
 # Local model
-*.mar
+*.nc.zip
 
 logs/
 examples/logs/

CHANGELOG.md

@@ -15,11 +15,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `nc.list_my_orgs()` method to find the details of your organizations
 - Support for Python 3.9
 - Added Pi0
+- Ability to launch a policy outside of a server using `nc.policy()`
 
 ### Changed
 
 - The current organization is now stored locally at `~/.neuracore/config.json` rather than being set globally
 - Training now supports all data types. See CNNMLP for an example.
+- Endpoints now use our own custom server, rather than torchserve
+- `nc.connect_local_endpoint()` -> `nc.policy_local_server()`
+- `nc.connect_endpoint()` -> `nc.policy_remote_server()`
+- Moved from .mar format to .nc.zip format for model archives
 
 ### Removed
 

README.md

@@ -68,6 +68,37 @@ nc.log_joint_positions({
 nc.log_rgb("top_camera", image_array)
 ```
 
+## Command Line Commands
+
+Neuracore provides several command-line tools for authentication, organization management, and server operations:
+
+### Authentication
+```bash
+# Generate and save API key (interactive login)
+nc-login
+```
+
+### Organization Management
+```bash
+# Select current organization (interactive selection)
+nc-select-org
+```
+
+### Server Operations
+```bash
+# Launch local policy server
+nc-launch-server --job_id <job_id> --org_id <org_id> [--host <host>] [--port <port>]
+
+# Example:
+nc-launch-server --job_id my_job_123 --org_id my_org_456 --host 0.0.0.0 --port 8080
+```
+
+**Parameters:**
+- `--job_id`: Required. The job ID to run
+- `--org_id`: Required. Your organization ID
+- `--host`: Optional. Host address (default: 0.0.0.0)
+- `--port`: Optional. Port number (default: 8080)
+
 ## Open Source Training
 
 Neuracore includes a powerful open-source training infrastructure built with Hydra for configuration management. Train your own robot learning algorithms locally rather than using our cloud training service.

examples/README.md

@@ -80,22 +80,6 @@ For local model deployment, you'll need additional packages:
 pip install "neuracore[ml]"
 ```
 
-Local model deployment also requires a Java JRE.
-
-<details>
-<summary>Install Java JRE</summary>
-
-Mac:
-```bash
-brew install temurin java
-```
-
-Linux:
-```bash
-sudo apt install default-jre
-```
-</details>
-
 
 Run the local model:
 ```bash

examples/example_local_endpoint.py

@@ -17,9 +17,17 @@ def main():
         overwrite=False,
     )
     # If you have a train run name, you can use it to connect to a local. E.g.:
-    policy = nc.connect_local_endpoint(train_run_name=TRAINING_JOB_NAME)
-    # If you know the path to the local model.mar file, you can use it directly as:
-    # policy = nc.connect_local_endpoint(path_to_model="PATH/TO/MODEL.mar")
+    policy = nc.policy(train_run_name=TRAINING_JOB_NAME)
+
+    # If you know the path to the local model.nc.zip file, you can use it directly as:
+    # policy = nc.policy(model_file=PATH/TO/MODEL.nc.zip)
+
+    # Alternatively, you can connect to a local endpoint that has been started
+    # policy = nc.policy_local_server(train_run_name=TRAINING_JOB_NAME)
+
+    # Optional. Set the checkpoint to the last epoch.
+    # Note by default, model is loaded from the last epoch.
+    policy.set_checkpoint(epoch=-1)
 
     onscreen_render = True
     render_cam_name = "angle"

examples/example_server_endpoint.py

@@ -22,7 +22,7 @@ def main():
     )
 
     try:
-        policy = nc.connect_endpoint(ENDPOINT_NAME)
+        policy = nc.policy_remote_server(ENDPOINT_NAME)
     except EndpointError:
         print(f"Please ensure that the endpoint '{ENDPOINT_NAME}' is running.")
         print(

neuracore/__init__.py

@@ -7,4 +7,4 @@
 from .api.training import *  # noqa: F403
 from .core.exceptions import *  # noqa: F403
 
-__version__ = "1.6.6"
+__version__ = "2.0.0"

neuracore/api/endpoints.py

@@ -14,85 +14,100 @@
 
 from ..core.auth import get_auth
 from ..core.const import API_URL
-from ..core.endpoint import EndpointPolicy
-from ..core.endpoint import connect_endpoint as _connect_endpoint
-from ..core.endpoint import connect_local_endpoint as _connect_local_endpoint
+from ..core.endpoint import DirectPolicy, LocalServerPolicy, RemoteServerPolicy
+from ..core.endpoint import policy as _policy
+from ..core.endpoint import policy_local_server as _policy_local_server
+from ..core.endpoint import policy_remote_server as _policy_remote_server
 
 
-def connect_endpoint(
-    endpoint_name: str, robot_name: Optional[str] = None, instance: int = 0
-) -> EndpointPolicy:
-    """Connect to a deployed model endpoint for inference.
+def policy(
+    train_run_name: Optional[str] = None,
+    model_file: Optional[str] = None,
+    robot_name: Optional[str] = None,
+    instance: int = 0,
+) -> DirectPolicy:
+    """Launch a direct policy that runs the model in-process without any server.
 
-    Establishes a connection to a model endpoint that has been deployed on the
-    Neuracore platform. The endpoint can be used to make predictions with the
-    deployed model, and data logging is associated with the specified robot.
+    This is the fastest option with lowest latency since there's no network overhead.
+    The model runs directly in your Python process.
 
     Args:
-        endpoint_name: Name of the deployed endpoint to connect to.
+        train_run_name: Name of the training run to load the model from.
         robot_name: Robot name that predictions and data will be associated with.
             If not provided, uses the last initialized robot from global state.
         instance: Instance number of the robot for multi-instance deployments.
 
     Returns:
-        Policy object that provides an interface for making predictions
-        with the deployed model.
+        DirectPolicy object that provides direct in-process model inference.
 
     Raises:
-        EndpointError: If the endpoint connection fails due to invalid endpoint
-            name, authentication issues, or network problems.
-        ConfigError: If there is an error trying to get the current org
+        EndpointError: If the model download or initialization fails.
+        ConfigError: If there is an error trying to get the current org.
     """
-    return _connect_endpoint(
-        endpoint_name=endpoint_name, robot_name=robot_name, instance=instance
-    )
+    return _policy(train_run_name, model_file, robot_name, instance)
 
 
-def connect_local_endpoint(
-    path_to_model: Optional[str] = None,
+def policy_local_server(
     train_run_name: Optional[str] = None,
+    model_file: Optional[str] = None,
     port: int = 8080,
     robot_name: Optional[str] = None,
     instance: int = 0,
-) -> EndpointPolicy:
-    """Connect to a local model endpoint (run locally on your hardware).
+    host: str = "127.0.0.1",
+) -> LocalServerPolicy:
+    """Launch and connect to a local server policy.
 
-    Establishes a connection to a locally hosted model endpoint. The model can
-    be specified either by providing a direct path to a .mar model file or by
-    referencing a training run name. Only one of these options should be provided.
+    This option provides server-like architecture while maintaining local control.
 
     Args:
-        path_to_model: Direct file path to a local .mar (Model ARchive) model file.
-            Mutually exclusive with train_run_name.
-        train_run_name: Name of a training run to load the model from. The system
-            will locate and load the model from the specified training run.
-            Mutually exclusive with path_to_model.
-        port: TCP port number where the local endpoint is running.
+        train_run_name: Name of the training run to load the model from.
+        model_file: Path to the model file to load.
+        port: TCP port number where the local server will run.
         robot_name: Robot name that predictions and data will be associated with.
             If not provided, uses the last initialized robot from global state.
         instance: Instance number of the robot for multi-instance deployments.
+        host: Host address to bind the server to. Defaults to localhost.
 
     Returns:
-        Policy object that provides an interface for making predictions
-        with the local model.
+        LocalServerPolicy object that manages a local FastAPI server.
 
     Raises:
-        EndpointError: If the endpoint connection fails due to invalid model path,
-            inaccessible port, or conflicting parameters.
-        ValueError: If both path_to_model and train_run_name are provided, or if
-            neither is provided.
-        FileNotFoundError: If the specified model file doesn't exist.
-        ConfigError: If there is an error trying to get the current org
+        EndpointError: If the server startup or model initialization fails.
+        ConfigError: If there is an error trying to get the current org.
     """
-    return _connect_local_endpoint(
-        robot_name=robot_name,
-        instance=instance,
-        path_to_model=path_to_model,
-        train_run_name=train_run_name,
-        port=port,
+    return _policy_local_server(
+        train_run_name, model_file, port, robot_name, instance, host
     )
 
 
+def policy_remote_server(
+    endpoint_name: str,
+    robot_name: Optional[str] = None,
+    instance: int = 0,
+) -> RemoteServerPolicy:
+    """Connects to a policy that is remotely running on neuracore.
+
+    Connects to a model endpoint deployed on the Neuracore cloud platform.
+    The endpoint must be active and accessible.
+
+    Args:
+        endpoint_name: Name of the deployed endpoint to connect to.
+        robot_name: Robot name that predictions and data will be associated with.
+            If not provided, uses the last initialized robot from global state.
+        instance: Instance number of the robot for multi-instance deployments.
+
+    Returns:
+        RemoteServerPolicy object for making predictions with the remote endpoint.
+
+    Raises:
+        EndpointError: If the endpoint connection fails due to invalid endpoint
+            name, authentication issues, or network problems.
+        ConfigError: If there is an error trying to get the current org.
+    """
+    return _policy_remote_server(endpoint_name, robot_name, instance)
+
+
+# Deployment management functions
 def deploy_model(job_id: str, name: str) -> dict:
     """Deploy a trained model to a managed endpoint.
 

neuracore/core/cli/launch_server.py

@@ -0,0 +1,74 @@
+"""Algorithm validation script for neuracore ML algorithms.
+
+This module provides a command-line tool for validating ML algorithms in an
+isolated virtual environment. It creates a temporary venv, installs dependencies,
+and runs validation to ensure algorithms meet neuracore requirements.
+"""
+
+import argparse
+import logging
+
+import neuracore as nc
+from neuracore.core.endpoint import policy_local_server
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    """Main entry point for the neuracore-validate command-line tool.
+
+    Parses command-line arguments, validates the provided algorithm folder,
+    and exits with appropriate status code.
+
+    Usage:
+        neuracore-validate <path_to_algorithm_folder>
+        neuracore-validate --algorithm_folder <path_to_algorithm_folder>
+        neuracore-validate --algorithm_id <algorithm_id>
+
+    Exit codes:
+        0: Validation succeeded
+        1: Validation failed or invalid arguments
+    """
+    parser = argparse.ArgumentParser(
+        description="Validate neuracore ML algorithms",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--job_id",
+        type=str,
+        help="Job ID to run",
+    )
+    parser.add_argument(
+        "--org_id",
+        type=str,
+        help="Organization ID",
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="0.0.0.0",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8080,
+    )
+
+    args = parser.parse_args()
+
+    nc.login()
+    nc.set_organization(args.org_id)
+    policy = policy_local_server(
+        train_run_name="",  # Use job id instead
+        port=args.port,
+        host=args.host,
+        job_id=args.job_id,
+    )
+    assert policy.server_process is not None
+    policy.server_process.wait()
+
+
+if __name__ == "__main__":
+    main()