diff --git a/sdk_v2/rust/.clippy.toml b/sdk_v2/rust/.clippy.toml new file mode 100644 index 000000000..1d42f2f12 --- /dev/null +++ b/sdk_v2/rust/.clippy.toml @@ -0,0 +1,2 @@ +# Clippy configuration for Foundry Local Rust SDK +msrv = "1.70" diff --git a/sdk_v2/rust/.rustfmt.toml b/sdk_v2/rust/.rustfmt.toml new file mode 100644 index 000000000..dce363edf --- /dev/null +++ b/sdk_v2/rust/.rustfmt.toml @@ -0,0 +1,3 @@ +edition = "2021" +max_width = 100 +use_field_init_shorthand = true diff --git a/sdk_v2/rust/Cargo.toml b/sdk_v2/rust/Cargo.toml new file mode 100644 index 000000000..c73a2476c --- /dev/null +++ b/sdk_v2/rust/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "foundry-local-sdk" +version = "2.0.0" +edition = "2021" +license = "MIT" +readme = "README.md" +description = "Local AI model inference powered by the Foundry Local Core engine" +homepage = "https://www.foundrylocal.ai/" +repository = "https://github.com/microsoft/Foundry-Local" +documentation = "https://github.com/microsoft/Foundry-Local/blob/main/sdk_v2/rust/docs/api.md" +include = ["src/**", "build.rs", "Cargo.toml", "README.md", "LICENSE.txt", "deps_versions.json"] + +[features] +default = [] +winml = [] +nightly = [] + +[dependencies] +libloading = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } +tokio-stream = "0.1" +tokio-util = "0.7" +futures-core = "0.3" +reqwest = { version = "0.12", features = ["json"] } +urlencoding = "2" +async-openai = { version = "=0.33.1", default-features = false, features = ["chat-completion-types", "embedding-types"] } + +[build-dependencies] +ureq = "3" +zip = "2" +serde_json = "1" +serde = { version = "1", features = ["derive"] } + +[[example]] +name = "chat_completion" +path = "examples/chat_completion.rs" + +[[example]] +name = "tool_calling" +path = "examples/tool_calling.rs" + +[[example]] +name = "interactive_chat" +path = "examples/interactive_chat.rs" + +[[example]] +name = "low_level_inference" +path = "examples/low_level_inference.rs" + +[lints.clippy] +all = { level = "warn", priority = -1 } diff --git a/sdk_v2/rust/GENERATE-DOCS.md b/sdk_v2/rust/GENERATE-DOCS.md new file mode 100644 index 000000000..6f7485124 --- /dev/null +++ b/sdk_v2/rust/GENERATE-DOCS.md @@ -0,0 +1,41 @@ +# Generating API Reference Docs + +The Rust SDK uses `cargo doc` to generate API documentation from `///` doc comments in the source code. + +## Viewing Docs Locally + +To generate and open the API docs in your browser: + +```bash +cd sdk_v2/rust +cargo doc --no-deps --open +``` + +This generates HTML documentation at `target/doc/foundry_local_sdk/index.html`. + +## Public API Surface + +The SDK re-exports all public types from the crate root. Key modules: + +| Module / Type | Description | +|---|---| +| `FoundryLocalManager` | Entry point — SDK initialisation, web service lifecycle | +| `FoundryLocalConfig` | Configuration (app name, log level, service endpoint) | +| `Catalog` | Model discovery and lookup | +| `Model` | Grouped model (alias → best variant) | +| `DownloadBuilder` | Builder for model downloads (progress, cancellation) | +| `ChatClient` | OpenAI-compatible chat completions (sync + streaming) | +| `AudioClient` | OpenAI-compatible audio transcription (sync + streaming) | +| `CreateChatCompletionResponse` | Typed chat completion response (from `async-openai`) | +| `CreateChatCompletionStreamResponse` | Typed streaming chat chunk (from `async-openai`) | +| `AudioTranscriptionResponse` | Typed audio transcription response | +| `FoundryLocalError` | Error enum with variants for all failure modes | + +## Notes + +- Unlike the C# and JS SDKs which commit generated markdown docs, Rust's convention is to generate HTML docs on demand with `cargo doc`. +- Once the crate is published to crates.io, docs will be automatically hosted at [docs.rs](https://docs.rs). +- Use `--document-private-items` to include internal/private API in the generated docs: + ```bash + cargo doc --no-deps --document-private-items --open + ``` diff --git a/sdk_v2/rust/LICENSE.txt b/sdk_v2/rust/LICENSE.txt new file mode 100644 index 000000000..48bc6bb49 --- /dev/null +++ b/sdk_v2/rust/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sdk_v2/rust/README.md b/sdk_v2/rust/README.md new file mode 100644 index 000000000..13d5a4dc6 --- /dev/null +++ b/sdk_v2/rust/README.md @@ -0,0 +1,608 @@ +# Foundry Local Rust SDK (v2) + +The Foundry Local Rust SDK provides an async Rust interface for running AI models locally on your machine. Discover, download, load, and run inference — all without cloud dependencies. + +> **v2 — built on the Foundry Local C++ engine.** This is the v2 binding, layered on the +> `foundry_local` C ABI (the same native engine used by the v2 Python and C# SDKs). Its public +> API is **fully backwards-compatible** with the v1 Rust SDK (`sdk/rust`) — existing code +> compiles and runs unchanged. + +## Features + +- **Local-first AI** — Run models entirely on your machine with no cloud calls +- **Model catalog** — Browse and discover available models; check what's cached or loaded +- **Automatic model management** — Download, load, unload, and remove models from cache +- **Chat completions** — OpenAI-compatible chat API with both non-streaming and streaming responses +- **Embeddings** — Generate text embeddings via OpenAI-compatible API +- **Audio transcription** — Transcribe audio files locally with streaming support +- **Tool calling** — Function/tool calling with streaming, multi-turn conversation support +- **Response format control** — Text, JSON, JSON Schema, and Lark grammar constrained output +- **Multi-variant models** — Models can have multiple variants (e.g., different quantizations) with automatic selection of the best cached variant +- **Embedded web service** — Start a local HTTP server for OpenAI-compatible API access +- **WinML support** — Automatic execution provider download on Windows for NPU/GPU acceleration +- **Configurable inference** — Control temperature, max tokens, top-k, top-p, frequency penalty, random seed, and more +- **Async-first** — Every operation is `async`; designed for use with the `tokio` runtime +- **Safe FFI** — Dynamically loads the native Foundry Local engine (`foundry_local`) with a safe Rust wrapper + +## Prerequisites + +- **Rust** 1.70+ (stable toolchain) +- The native `foundry_local` engine (plus its ONNX Runtime / GenAI dependencies) available at + build or run time — see [Native binary](#native-binary) + +## Installation + +```sh +cargo add foundry-local-sdk +``` + +Or add to your `Cargo.toml`: + +```toml +[dependencies] +foundry-local-sdk = "2" +``` + +You also need an async runtime. Most examples use [tokio](https://crates.io/crates/tokio): + +```toml +[dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tokio-stream = "0.1" # for StreamExt on streaming responses +``` + +### Feature Flags + +| Feature | Description | +|-----------|-------------| +| `winml` | Use the WinML backend (Windows only). Selects different ONNX Runtime and GenAI packages for NPU/GPU acceleration. | +| `nightly` | Resolve the latest nightly build of the Core package from the ORT-Nightly feed. | + +Enable features in `Cargo.toml`: + +```toml +[dependencies] +foundry-local-sdk = { version = "2", features = ["winml"] } +``` + +> **Note:** The `winml` feature is only relevant on Windows. On macOS and Linux, the standard build is used regardless. No code changes are needed — your application code stays the same. + +With `winml` enabled on Windows, the `winml` feature selects the WinML Runtime package (`Microsoft.AI.Foundry.Local.Runtime.WinML`) when downloading via `FOUNDRY_LOCAL_RUNTIME_VERSION`, and the WinML execution-provider DLLs are pre-loaded alongside `foundry_local`. See [Native binary](#native-binary) for how the engine is obtained. + +### Explicit EP Management + +You can explicitly discover and download execution providers: + +```rust +use foundry_local_sdk::{FoundryLocalConfig, FoundryLocalManager}; + +let manager = FoundryLocalManager::create(FoundryLocalConfig::new("my_app"))?; + +// Discover available EPs and their status +let eps = manager.discover_eps()?; +for ep in &eps { + println!("{} — registered: {}", ep.name, ep.is_registered); +} + +// Download and register all available EPs +let result = manager.download_and_register_eps(None).await?; +println!("Success: {}, Status: {}", result.success, result.status); + +// Download only specific EPs +let result = manager.download_and_register_eps(Some(&[eps[0].name.as_str()])).await?; +``` + +#### Per-EP download progress + +Use `download_and_register_eps_with_progress` to receive typed `(ep_name, percent)` updates +as each EP downloads (`percent` is 0.0–100.0): + +```rust +use std::sync::{Arc, Mutex}; + +let current_ep = Arc::new(Mutex::new(String::new())); +let ep = Arc::clone(¤t_ep); +manager.download_and_register_eps_with_progress(None, move |ep_name: &str, percent: f64| { + let mut current = ep.lock().unwrap(); + if ep_name != current.as_str() { + if !current.is_empty() { + println!(); + } + *current = ep_name.to_string(); + } + print!("\r {} {:5.1}%", ep_name, percent); +}).await?; +println!(); +``` + +#### Cancelling model and EP downloads + +Use a shared `Arc` with the download builders. Set the flag from another task or signal handler to stop the in-progress download. + +```rust +use std::sync::{ + atomic::AtomicBool, + Arc, +}; + +// manager and model already initialized +let cancel_flag = Arc::new(AtomicBool::new(false)); +// call cancel_flag.store(true, ...) from another task or signal handler to cancel + +manager + .download_and_register_eps_builder() + .cancel(Arc::clone(&cancel_flag)) + .run() + .await?; +model + .download_builder() + .cancel(Arc::clone(&cancel_flag)) + .run() + .await?; +``` + +Catalog access does not block on EP downloads. Call `download_and_register_eps` when you need hardware-accelerated execution providers. + +## Quick Start + +```rust +use foundry_local_sdk::{ + ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, + ChatCompletionRequestUserMessage, FoundryLocalConfig, FoundryLocalManager, +}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // 1. Initialize the manager — loads native libraries and starts the engine + let manager = FoundryLocalManager::create(FoundryLocalConfig::new("my_app"))?; + + // 2. Get a model from the catalog and load it + let model = manager.catalog().get_model("phi-3.5-mini").await?; + model.load().await?; + + // 3. Create a chat client and run inference + let client = model.create_chat_client() + .temperature(0.7) + .max_tokens(256); + + let messages: Vec = vec![ + ChatCompletionRequestSystemMessage::from("You are a helpful assistant.").into(), + ChatCompletionRequestUserMessage::from("What is the capital of France?").into(), + ]; + + let response = client.complete_chat(&messages, None).await?; + println!("{}", response.choices[0].message.content.as_deref().unwrap_or("")); + + // 4. Clean up + model.unload().await?; + + Ok(()) +} +``` + +## Usage + +### Browsing the Model Catalog + +The `Catalog` lets you discover what models are available, which are already cached locally, and which are currently loaded in memory. + +```rust +let catalog = manager.catalog(); + +// List all available models +let models = catalog.get_models().await?; +for model in &models { + println!("{} (id: {})", model.alias(), model.id()); +} + +// Look up a specific model by alias +let model = catalog.get_model("phi-3.5-mini").await?; + +// Look up a specific variant by its unique model ID +let variant = catalog.get_model_variant("phi-3.5-mini-generic-gpu-4").await?; + +// See what's already downloaded +let cached = catalog.get_cached_models().await?; + +// See what's currently loaded in memory +let loaded = catalog.get_loaded_models().await?; +``` + +### Model Lifecycle + +Each model may have multiple variants (different quantizations, hardware targets). The SDK auto-selects the best available variant, preferring cached versions. All models are represented by the `Model` type. + +```rust +let model = catalog.get_model("phi-3.5-mini").await?; + +// Inspect available variants +println!("Selected: {}", model.id()); +for v in model.variants() { + println!(" {} (info.cached: {})", v.id(), v.info().cached); +} +``` + +Download, load, and unload: + +```rust +// Download with progress reporting +model.download(Some(|progress: f64| { + print!("\r{progress:.1}%"); + std::io::Write::flush(&mut std::io::stdout()).ok(); +})).await?; + +// Or use the builder when combining progress, cancellation, or future options +let cancel_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); +model.download_builder() + .progress(|progress| { + print!("\r{progress:.1}%"); + std::io::Write::flush(&mut std::io::stdout()).ok(); + }) + .cancel(cancel_flag.clone()) + .run() + .await?; + +// Load into memory +model.load().await?; + +// Unload when done +model.unload().await?; + +// Remove from local cache entirely +model.remove_from_cache().await?; +``` + +### Chat Completions + +The `ChatClient` follows the OpenAI Chat Completion API structure. + +```rust +let client = model.create_chat_client() + +// Configure generation settings (fluent builder) + .temperature(0.7) + .max_tokens(256) + .top_p(0.9) + .frequency_penalty(0.5); + +// Non-streaming completion +let response = client.complete_chat( + &[ + ChatCompletionRequestSystemMessage::from("You are a helpful assistant.").into(), + ChatCompletionRequestUserMessage::from("Explain Rust's ownership model.").into(), + ], + None, +).await?; + +println!("{}", response.choices[0].message.content.as_deref().unwrap_or("")); +``` + +### Streaming Responses + +For real-time token-by-token output, use streaming: + +```rust +use tokio_stream::StreamExt; + +let mut stream = client.complete_streaming_chat( + &[ChatCompletionRequestUserMessage::from("Write a short poem about Rust.").into()], + None, +).await?; + +while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if let Some(content) = &chunk.choices[0].delta.content { + print!("{content}"); + } +} + +// Errors from the native core are delivered as stream items — +// no separate close() call needed. +``` + +### Tool Calling + +Define functions the model can call and handle the multi-turn conversation: + +```rust +use foundry_local_sdk::{ + ChatCompletionRequestMessage, ChatCompletionRequestToolMessage, + ChatCompletionTools, ChatFinishReason, ChatToolChoice, +}; +use serde_json::json; + +// Define available tools +let tools: Vec = serde_json::from_value(json!([{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": { "type": "string", "description": "City name" } + }, + "required": ["location"] + } + } +}]))?; + +let client = model.create_chat_client() + .max_tokens(512) + .tool_choice(ChatToolChoice::Auto); + +let mut messages: Vec = vec![ + ChatCompletionRequestUserMessage::from("What's the weather in Seattle?").into(), +]; + +// First request — model may call a tool +let response = client.complete_chat(&messages, Some(&tools)).await?; +let choice = &response.choices[0]; + +if choice.finish_reason == Some(ChatFinishReason::ToolCalls) { + if let Some(tool_calls) = &choice.message.tool_calls { + for tc in tool_calls { + // Execute the tool (your application logic) + let result = execute_tool(&tc.function.name, &tc.function.arguments); + + // Add assistant message with tool calls, then the tool result + messages.push(serde_json::from_value(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ "id": tc.id, "type": "function", + "function": { "name": tc.function.name, + "arguments": tc.function.arguments } }] + }))?); + messages.push(ChatCompletionRequestToolMessage { + content: result.into(), + tool_call_id: tc.id.clone(), + }.into()); + } + + // Continue the conversation with tool results + let final_response = client.complete_chat(&messages, Some(&tools)).await?; + println!("{}", final_response.choices[0].message.content.as_deref().unwrap_or("")); + } +} +``` + +Tool calling also works with streaming via `complete_streaming_chat` — accumulate tool call fragments during streaming and check for `ChatFinishReason::ToolCalls`. + +### Response Format Options + +Control the output format of chat completions: + +```rust +use foundry_local_sdk::ChatResponseFormat; + +// Plain text (default) +let client = model.create_chat_client() + .response_format(ChatResponseFormat::Text); + +// Unstructured JSON output +let client = model.create_chat_client() + .response_format(ChatResponseFormat::JsonObject); + +// JSON constrained to a schema +let client = model.create_chat_client() + .response_format(ChatResponseFormat::JsonSchema(r#"{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "integer" } + }, + "required": ["name", "age"] + }"#.to_string())); + +// Output constrained by a Lark grammar (Foundry extension) +let client = model.create_chat_client() + .response_format(ChatResponseFormat::LarkGrammar(grammar.to_string())); +``` + +### Embeddings + +Generate text embeddings using the `EmbeddingClient`: + +```rust +let embedding_client = model.create_embedding_client(); + +// Single input +let response = embedding_client + .generate_embedding("The quick brown fox jumps over the lazy dog") + .await?; +let embedding = &response.data[0].embedding; // Vec +println!("Dimensions: {}", embedding.len()); + +// Batch input +let batch_response = embedding_client + .generate_embeddings(&["The quick brown fox", "The capital of France is Paris"]) + .await?; +// batch_response.data[0].embedding, batch_response.data[1].embedding +``` + +### Audio Transcription + +Transcribe audio files locally using the `AudioClient`: + +```rust +let model = manager.catalog().get_model("whisper-tiny").await?; +model.load().await?; + +let audio_client = model.create_audio_client() + .language("en"); + +// Non-streaming transcription +let result = audio_client.transcribe("recording.wav").await?; +println!("{}", result.text); +``` + +#### Streaming Transcription + +```rust +use tokio_stream::StreamExt; + +let mut stream = audio_client.transcribe_streaming("recording.wav").await?; +while let Some(chunk) = stream.next().await { + print!("{}", chunk?.text); +} +``` + +### Embedded Web Service + +Start a local HTTP server that exposes an OpenAI-compatible REST API: + +```rust +manager.start_web_service().await?; +let urls = manager.urls()?; +println!("Service running at: {:?}", urls); + +// Any OpenAI-compatible client or tool can now connect to the endpoint. +// ... + +manager.stop_web_service().await?; +``` + +### Chat Client Settings + +All settings are configured via chainable builder methods on `ChatClient`: + +| Method | Type | Description | +|--------|------|-------------| +| `temperature(v)` | `f64` | Sampling temperature (0.0–2.0; higher = more random) | +| `max_tokens(v)` | `u32` | Maximum number of tokens to generate | +| `top_p(v)` | `f64` | Nucleus sampling probability (0.0–1.0) | +| `top_k(v)` | `u32` | Top-k sampling parameter (Foundry extension) | +| `frequency_penalty(v)` | `f64` | Frequency penalty | +| `presence_penalty(v)` | `f64` | Presence penalty | +| `n(v)` | `u32` | Number of completions to generate | +| `random_seed(v)` | `u64` | Random seed for reproducible results (Foundry extension) | +| `response_format(v)` | `ChatResponseFormat` | Output format (Text, JsonObject, JsonSchema, LarkGrammar) | +| `tool_choice(v)` | `ChatToolChoice` | Tool selection strategy (None, Auto, Required, Function) | + +## Error Handling + +All fallible operations return `foundry_local_sdk::Result`, which is an alias for `std::result::Result`. + +```rust +use foundry_local_sdk::FoundryLocalError; + +match manager.catalog().get_model("nonexistent").await { + Ok(model) => { /* use model */ } + Err(FoundryLocalError::ModelOperation { reason }) => { + eprintln!("Model error: {reason}"); + } + Err(FoundryLocalError::CommandExecution { reason }) => { + eprintln!("Core engine error: {reason}"); + } + Err(e) => { + eprintln!("Unexpected error: {e}"); + } +} +``` + +### Error Variants + +| Variant | Description | +|---------|-------------| +| `LibraryLoad { reason }` | The native core library could not be loaded | +| `CommandExecution { reason }` | A command executed against native core returned an error | +| `InvalidConfiguration { reason }` | The provided configuration is invalid | +| `ModelOperation { reason }` | A model operation failed (load, unload, download, etc.) | +| `HttpRequest(reqwest::Error)` | An HTTP request to an external service failed | +| `Serialization(serde_json::Error)` | JSON serialization/deserialization failed | +| `Validation { reason }` | A validation check on user-supplied input failed | +| `Io(std::io::Error)` | An I/O error occurred | +| `Internal { reason }` | An internal SDK error (e.g. poisoned lock) | + +## Configuration + +The SDK is configured via `FoundryLocalConfig` when creating the manager: + +```rust +use foundry_local_sdk::{FoundryLocalConfig, LogLevel}; + +let config = FoundryLocalConfig::new("my_app") + .log_level(LogLevel::Info) + .model_cache_dir("/path/to/cache") + .web_service_urls("http://127.0.0.1:5000"); + +let manager = FoundryLocalManager::create(config)?; +``` + +| Setting | Builder method | Default | Description | +|---------|---------------|---------|-------------| +| App name | `new(name)` | **(required)** | Your application name | +| App data dir | `.app_data_dir(dir)` | `~/.{app_name}` | Application data directory | +| Model cache dir | `.model_cache_dir(dir)` | `{app_data_dir}/cache/models` | Where models are stored locally | +| Logs dir | `.logs_dir(dir)` | `{app_data_dir}/logs` | Log output directory | +| Log level | `.log_level(level)` | `Warn` | `Trace`, `Debug`, `Info`, `Warn`, `Error`, `Fatal` | +| Web service URLs | `.web_service_urls(urls)` | `None` | Bind address for the embedded web service | +| Service endpoint | `.service_endpoint(url)` | `None` | URL of an existing external service to connect to | +| Library path | `.library_path(path)` | Auto-discovered | Path to the native `foundry_local` library (or its directory) | +| Additional settings | `.additional_setting(k, v)` | `None` | Extra key-value settings passed to Core | +| Logger | `.logger(impl Logger)` | `None` | Application logger (stub — not yet wired) | + +## How It Works + +### Native binary + +The SDK loads the `foundry_local` native library (and its ONNX Runtime / GenAI dependencies) at +runtime. The `build.rs` build script can obtain it in two ways, controlled by environment variables: + +| Variable | Purpose | +|----------|---------| +| `FOUNDRY_LOCAL_NATIVE_BIN_DIR` | Copy native binaries from a local C++ build output directory (the dev path). Mirrors the C# `FoundryLocalNativeBinDir`. | +| `FOUNDRY_LOCAL_RUNTIME_VERSION` | Download the Runtime NuGet package (`Microsoft.AI.Foundry.Local.Runtime`, or `.Runtime.WinML` with the `winml` feature) plus ONNX Runtime / GenAI for the target RID. | + +If neither is set at build time, the library is resolved at **runtime** from (in order): + +1. `FoundryLocalConfig::library_path` — a path to the `foundry_local` library file or its directory. +2. The `FOUNDRY_LOCAL_LIB_DIR` environment variable. +3. The directory of the running executable. +4. The system loader search path. + +> **Migration note:** in v1, `FoundryLocalConfig::library_path` pointed at the +> `Microsoft.AI.Foundry.Local.Core` library. In v2 it points at the `foundry_local` library (or its +> containing directory). The builder method and signature are unchanged. + +ONNX Runtime and GenAI are pre-loaded before `foundry_local` so the dynamic loader resolves the +engine's dependencies regardless of rpath/search-path setup. On platforms where GenAI honours it, +`ORT_LIB_PATH` is set to the pre-loaded ONNX Runtime. + +### Runtime Loading + +At runtime, the SDK uses `libloading` to dynamically load the `foundry_local` library, resolve the +API function table via `FoundryLocalGetApi`, and cache the sub-API tables. No static linking or +system-wide installation is required. + +## Platform Support + +| Platform | RID | Status | +|-----------------|--------------|--------| +| Windows x64 | `win-x64` | ✅ | +| Windows ARM64 | `win-arm64` | ✅ | +| Linux x64 | `linux-x64` | ✅ | +| Linux ARM64 | `linux-arm64`| ✅ | +| macOS ARM64 | `osx-arm64` | ✅ | + +## Running Examples + +Sample applications are available in [`samples/rust/`](../../samples/rust/): + +| Sample | Description | +|--------|-------------| +| `native-chat-completions` | Non-streaming and streaming chat completions | +| `tool-calling-foundry-local` | Function/tool calling with multi-turn conversations | +| `audio-transcription-example` | Audio transcription (non-streaming and streaming) | +| `foundry-local-webserver` | Embedded OpenAI-compatible REST API server | + +Run a sample with: + +```sh +cd samples/rust +cargo run -p native-chat-completions +``` + +## License + +Microsoft Software License Terms — see [LICENSE](../../LICENSE) for details. diff --git a/sdk_v2/rust/build.rs b/sdk_v2/rust/build.rs new file mode 100644 index 000000000..41edfbf3f --- /dev/null +++ b/sdk_v2/rust/build.rs @@ -0,0 +1,324 @@ +//! Build script for the Foundry Local v2 Rust SDK. +//! +//! Obtains the native `foundry_local` library (plus ONNX Runtime + GenAI) and +//! makes it discoverable at runtime via the `FOUNDRY_NATIVE_DIR` compile-time +//! env that `detail::api` consults. +//! +//! Native acquisition order: +//! 1. `FOUNDRY_LOCAL_NATIVE_BIN_DIR` — copy native files from a local C++ build +//! (the dev path, mirroring the C# `FoundryLocalNativeBinDir`). +//! 2. `FOUNDRY_LOCAL_RUNTIME_VERSION` — download the Runtime NuGet package +//! (`Microsoft.AI.Foundry.Local.Runtime[.WinML]`) plus ORT/GenAI for the RID. +//! 3. Otherwise no-op: the library is resolved at runtime from +//! `FOUNDRY_LOCAL_LIB_DIR`, next to the executable, or the system path. + +use std::env; +use std::fs; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +const FEEDS: &[&str] = &[ + "https://api.nuget.org/v3/index.json", + "https://pkgs.dev.azure.com/aiinfra/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json", +]; + +struct DepsVersions { + ort: String, + genai: String, +} + +fn load_deps_versions(manifest_dir: &Path) -> DepsVersions { + let candidates = [ + manifest_dir.join("deps_versions.json"), + manifest_dir.join("..").join("deps_versions.json"), + ]; + let json_path = candidates + .iter() + .find(|p| p.exists()) + .cloned() + .unwrap_or_else(|| candidates[0].clone()); + println!("cargo:rerun-if-changed={}", json_path.display()); + + let content = fs::read_to_string(&json_path).unwrap_or_default(); + let stripped = content.strip_prefix('\u{FEFF}').unwrap_or(&content); + let val: serde_json::Value = serde_json::from_str(stripped).unwrap_or(serde_json::Value::Null); + let s = |key: &str| -> String { + val.get(key) + .and_then(|o| o.get("version")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string() + }; + DepsVersions { + ort: s("onnxruntime"), + genai: s("onnxruntime-genai"), + } +} + +fn get_rid() -> Option<&'static str> { + match (env::consts::OS, env::consts::ARCH) { + ("windows", "x86_64") => Some("win-x64"), + ("windows", "aarch64") => Some("win-arm64"), + ("linux", "x86_64") => Some("linux-x64"), + ("linux", "aarch64") => Some("linux-arm64"), + ("macos", "aarch64") => Some("osx-arm64"), + ("macos", "x86_64") => Some("osx-x64"), + _ => None, + } +} + +fn native_lib_extension() -> &'static str { + match env::consts::OS { + "windows" => "dll", + "macos" => "dylib", + _ => "so", + } +} + +struct NuGetPackage { + name: String, + version: String, + expected_file: String, +} + +fn get_packages(deps: &DepsVersions, runtime_version: &str) -> Vec { + let ext = native_lib_extension(); + let prefix = if env::consts::OS == "windows" { + "" + } else { + "lib" + }; + let runtime_name = if env::var("CARGO_FEATURE_WINML").is_ok() { + "Microsoft.AI.Foundry.Local.Runtime.WinML" + } else { + "Microsoft.AI.Foundry.Local.Runtime" + }; + + vec![ + NuGetPackage { + name: runtime_name.to_string(), + version: runtime_version.to_string(), + expected_file: format!("{prefix}foundry_local.{ext}"), + }, + NuGetPackage { + name: "Microsoft.ML.OnnxRuntime.Foundry".to_string(), + version: deps.ort.clone(), + expected_file: format!("{prefix}onnxruntime.{ext}"), + }, + NuGetPackage { + name: "Microsoft.ML.OnnxRuntimeGenAI.Foundry".to_string(), + version: deps.genai.clone(), + expected_file: format!("{prefix}onnxruntime-genai.{ext}"), + }, + ] +} + +fn resolve_base_address(feed_url: &str) -> Result { + let body: String = ureq::get(feed_url) + .call() + .map_err(|e| format!("fetch feed index {feed_url}: {e}"))? + .body_mut() + .read_to_string() + .map_err(|e| format!("read feed index: {e}"))?; + let index: serde_json::Value = + serde_json::from_str(&body).map_err(|e| format!("parse feed index: {e}"))?; + for resource in index["resources"].as_array().ok_or("missing resources")? { + if resource["@type"].as_str() == Some("PackageBaseAddress/3.0.0") { + if let Some(id) = resource["@id"].as_str() { + return Ok(if id.ends_with('/') { + id.to_string() + } else { + format!("{id}/") + }); + } + } + } + Err(format!("no PackageBaseAddress in {feed_url}")) +} + +fn try_download( + pkg: &NuGetPackage, + rid: &str, + out_dir: &Path, + feed_url: &str, +) -> Result { + let base = resolve_base_address(feed_url)?; + let name = pkg.name.to_lowercase(); + let version = pkg.version.to_lowercase(); + let url = format!("{base}{name}/{version}/{name}.{version}.nupkg"); + println!("cargo:warning=Downloading {} {}", pkg.name, pkg.version); + + let mut response = ureq::get(&url) + .call() + .map_err(|e| format!("download {}: {e}", pkg.name))?; + let mut bytes = Vec::new(); + response + .body_mut() + .as_reader() + .read_to_end(&mut bytes) + .map_err(|e| format!("read body {}: {e}", pkg.name))?; + + // Integrity guard: reject an empty/truncated payload before we try to treat + // it as a nupkg archive. + if bytes.is_empty() { + return Err(format!("downloaded {} {} is empty", pkg.name, pkg.version)); + } + + let ext = native_lib_extension(); + let native_prefix = format!("runtimes/{rid}/native/"); + let runtime_prefix = format!("runtimes/{rid}/"); + let mut archive = zip::ZipArchive::new(io::Cursor::new(&bytes)) + .map_err(|e| format!("open nupkg {}: {e}", pkg.name))?; + + let mut extracted = 0usize; + for i in 0..archive.len() { + let mut file = archive.by_index(i).map_err(|e| format!("zip entry: {e}"))?; + let entry = file.name().to_string(); + if !entry.ends_with(&format!(".{ext}")) { + continue; + } + let direct = entry + .strip_prefix(&runtime_prefix) + .map(|r| !r.is_empty() && !r.contains('/')) + .unwrap_or(false); + if !entry.starts_with(&native_prefix) && !direct { + continue; + } + let file_name = match Path::new(&entry).file_name() { + Some(n) => n.to_string_lossy().to_string(), + None => continue, + }; + let dest = out_dir.join(&file_name); + let mut out = + fs::File::create(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?; + io::copy(&mut file, &mut out).map_err(|e| format!("write {}: {e}", dest.display()))?; + println!("cargo:warning= Extracted {file_name}"); + extracted += 1; + } + Ok(extracted) +} + +fn download_and_extract(pkg: &NuGetPackage, rid: &str, out_dir: &Path) -> Result<(), String> { + if out_dir.join(&pkg.expected_file).exists() { + return Ok(()); + } + if pkg.version.trim().is_empty() { + return Err(format!("no version configured for {}", pkg.name)); + } + let mut last = String::new(); + for feed in FEEDS { + match try_download(pkg, rid, out_dir, feed) { + // Integrity guard: a "successful" download must actually yield the + // expected native binary (the archive opened as a valid zip and the + // expected file landed on disk). Cryptographic SHA-512 verification + // against the feed's published packageHash is feed-specific — the + // registration layout differs between nuget.org and the Azure DevOps + // feed — and is left as future hardening. + Ok(_) if out_dir.join(&pkg.expected_file).exists() => return Ok(()), + Ok(_) => { + last = format!( + "{} {} downloaded but did not contain expected file {}", + pkg.name, pkg.version, pkg.expected_file + ) + } + Err(e) => last = e, + } + } + Err(format!( + "download {} {} failed: {last}", + pkg.name, pkg.version + )) +} + +fn copy_from_local_dir(src: &Path, out_dir: &Path) -> bool { + let ext = native_lib_extension(); + let Ok(entries) = fs::read_dir(src) else { + return false; + }; + let mut copied = false; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some(ext) { + if let Some(name) = path.file_name() { + if fs::copy(&path, out_dir.join(name)).is_ok() { + println!( + "cargo:warning=Copied {} from FOUNDRY_LOCAL_NATIVE_BIN_DIR", + name.to_string_lossy() + ); + copied = true; + } + } + } + } + copied +} + +fn emit_native_dir(out_dir: &Path) { + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-env=FOUNDRY_NATIVE_DIR={}", out_dir.display()); + #[cfg(windows)] + println!("cargo:rustc-link-lib=kernel32"); +} + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=FOUNDRY_LOCAL_NATIVE_BIN_DIR"); + println!("cargo:rerun-if-env-changed=FOUNDRY_LOCAL_RUNTIME_VERSION"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_WINML"); + + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap_or_default()); + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); + + // 1. Local C++ build output (dev path). + if let Ok(local) = env::var("FOUNDRY_LOCAL_NATIVE_BIN_DIR") { + let src = Path::new(&local); + if src.is_dir() && copy_from_local_dir(src, &out_dir) { + emit_native_dir(&out_dir); + return; + } + } + + // 2. Runtime NuGet download (release path), only when a version is pinned. + let runtime_version = env::var("FOUNDRY_LOCAL_RUNTIME_VERSION").unwrap_or_default(); + if !runtime_version.trim().is_empty() { + let rid = match get_rid() { + Some(r) => r, + None => { + println!( + "cargo:warning=Unsupported platform {} {}; skipping native download.", + env::consts::OS, + env::consts::ARCH + ); + return; + } + }; + let deps = load_deps_versions(&manifest_dir); + let packages = get_packages(&deps, &runtime_version); + let mut failed = false; + for pkg in &packages { + if let Err(e) = download_and_extract(pkg, rid, &out_dir) { + println!("cargo:warning={e}"); + failed = true; + } + } + if !failed { + emit_native_dir(&out_dir); + } + return; + } + + // 3. No build-time native configured. Runtime discovery handles loading: + // FOUNDRY_LOCAL_LIB_DIR, the executable's directory, or the system loader + // path (see detail::api::resolve_library_path). Stay quiet when the runtime + // override is set — the `FOUNDRY_LOCAL_LIB_DIR=... cargo run` workflow is a + // fully supported path and shouldn't trigger a build warning. Only hint when + // nothing is configured at build *or* run time. + println!("cargo:rerun-if-env-changed=FOUNDRY_LOCAL_LIB_DIR"); + if env::var_os("FOUNDRY_LOCAL_LIB_DIR").is_none() { + println!( + "cargo:warning=foundry-local-sdk: no native library configured. Provide it at build time \ + via FOUNDRY_LOCAL_NATIVE_BIN_DIR (local C++ build) or FOUNDRY_LOCAL_RUNTIME_VERSION (NuGet), \ + or at runtime via FOUNDRY_LOCAL_LIB_DIR / by placing foundry_local on the loader search path." + ); + } +} diff --git a/sdk_v2/rust/docs/api.md b/sdk_v2/rust/docs/api.md new file mode 100644 index 000000000..dcf0bb05d --- /dev/null +++ b/sdk_v2/rust/docs/api.md @@ -0,0 +1,884 @@ +# Foundry Local Rust SDK — Public API Reference + +> Auto-generated from `sdk_v2/rust/src` source files. + +## Table of Contents + +- [Entry Point](#entry-point) + - [FoundryLocalManager](#foundrylocalmanager) + - [FoundryLocalConfig](#foundrylocalconfig) + - [Logger](#logger) + - [LogLevel](#loglevel) +- [Model Catalog](#model-catalog) + - [Catalog](#catalog) + - [Model](#model) +- [OpenAI Clients](#openai-clients) + - [ChatClient](#chatclient) + - [ChatCompletionStream](#chatcompletionstream) + - [EmbeddingClient](#embeddingclient) + - [EmbeddingResponse](#embeddingresponse) + - [AudioClient](#audioclient) + - [AudioTranscriptionStream](#audiotranscriptionstream) + - [AudioTranscriptionResponse](#audiotranscriptionresponse) + - [TranscriptionSegment](#transcriptionsegment) + - [TranscriptionWord](#transcriptionword) + - [JsonStream\](#jsonstreamt) +- [Inference API](#inference-api) + - [Session](#session) + - [ChatSession](#chatsession) + - [EmbeddingsSession](#embeddingssession) + - [AudioSession](#audiosession) + - [ItemQueue](#itemqueue) + - [ItemStream](#itemstream) + - [Item](#item) + - [Message](#message) + - [Tensor](#tensor) + - [Request](#request) + - [RequestOptions](#requestoptions) + - [Response](#response) + - [FinishReason](#finishreason) + - [ToolDefinition](#tooldefinition) +- [Types](#types) + - [ModelInfo](#modelinfo) + - [ChatResponseFormat](#chatresponseformat) + - [ChatToolChoice](#chattoolchoice) + - [DeviceType](#devicetype) + - [PromptTemplate](#prompttemplate) + - [Runtime](#runtime) + - [ModelSettings](#modelsettings) + - [Parameter](#parameter) +- [Error Handling](#error-handling) + - [FoundryLocalError](#foundrylocalerror) +- [Re-exported OpenAI Types](#re-exported-openai-types) + +--- + +## Entry Point + +### FoundryLocalManager + +Primary entry point for interacting with Foundry Local. Shared instance — while any handle is alive, `create()` returns the same instance; it is torn down when the last handle is dropped. + +```rust +pub struct FoundryLocalManager { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create` | `fn create(config: FoundryLocalConfig) -> Result, FoundryLocalError>` | Initialise the SDK. Returns a shared handle: while any handle is alive, all calls return the same instance (config ignored after the first). Once the last handle is dropped the native manager is torn down via `Drop`, and a later call builds a fresh instance. | +| `catalog` | `fn catalog(&self) -> &Catalog` | Access the model catalog. | +| `urls` | `fn urls(&self) -> Result, FoundryLocalError>` | URLs the local web service is listening on. Empty until `start_web_service` is called. | +| `start_web_service` | `async fn start_web_service(&self) -> Result<(), FoundryLocalError>` | Start the local web service. Retrieve listening URLs via `urls()`. | +| `stop_web_service` | `async fn stop_web_service(&self) -> Result<(), FoundryLocalError>` | Stop the local web service. | + +--- + +### FoundryLocalConfig + +User-facing configuration for initializing the SDK. Fields are private; use +the builder methods to customise. + +```rust +pub struct FoundryLocalConfig { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `fn new(app_name: impl Into) -> Self` | Create a new configuration. All optional fields default to `None`. | +| `app_data_dir` | `fn app_data_dir(self, dir: impl Into) -> Self` | Override the application-data directory. | +| `model_cache_dir` | `fn model_cache_dir(self, dir: impl Into) -> Self` | Override the model-cache directory. | +| `logs_dir` | `fn logs_dir(self, dir: impl Into) -> Self` | Override the logs directory. | +| `log_level` | `fn log_level(self, level: LogLevel) -> Self` | Set the log level. | +| `web_service_urls` | `fn web_service_urls(self, urls: impl Into) -> Self` | Set the web-service listen URLs. | +| `service_endpoint` | `fn service_endpoint(self, endpoint: impl Into) -> Self` | Set an external service endpoint URL. | +| `library_path` | `fn library_path(self, path: impl Into) -> Self` | Override the path to the native core library. | +| `additional_setting` | `fn additional_setting(self, key: impl Into, value: impl Into) -> Self` | Add a key-value pair to additional settings. | +| `logger` | `fn logger(self, logger: impl Logger + 'static) -> Self` | Provide an application logger (stub — not yet wired into native core). | + +**Example:** +```rust +let config = FoundryLocalConfig::new("my_app") + .log_level(LogLevel::Debug) + .model_cache_dir("/path/to/cache"); +``` + +--- + +### LogLevel + +```rust +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, + Fatal, +} +``` + +--- + +### Logger + +Application logger trait. Implement this to receive SDK log messages. + +> **Note:** Stub — not yet wired into the native core. Stored in configuration for future use. + +```rust +pub trait Logger: Send + Sync { + fn log(&self, level: LogLevel, message: &str); +} +``` + +--- + +### Catalog + +Discovers, caches, and looks up available models. + +```rust +pub struct Catalog { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `name` | `fn name(&self) -> &str` | Catalog name as reported by the native core. | +| `update_models` | `async fn update_models(&self) -> Result<(), FoundryLocalError>` | Refresh catalog if cache expired or invalidated. | +| `get_models` | `async fn get_models(&self) -> Result>, FoundryLocalError>` | Return all known models. | +| `get_model` | `async fn get_model(&self, alias: &str) -> Result, FoundryLocalError>` | Look up a model by alias. | +| `get_model_variant` | `async fn get_model_variant(&self, id: &str) -> Result, FoundryLocalError>` | Look up a variant by unique id. | +| `get_cached_models` | `async fn get_cached_models(&self) -> Result>, FoundryLocalError>` | Return only variants cached on disk. | +| `get_loaded_models` | `async fn get_loaded_models(&self) -> Result>, FoundryLocalError>` | Return model variants currently loaded in memory. | + +--- + +### Model + +Groups one or more variants sharing the same alias. By default, the cached variant is selected. + +```rust +pub struct Model { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `alias` | `fn alias(&self) -> &str` | Alias shared by all variants. | +| `id` | `fn id(&self) -> &str` | Unique identifier of the selected variant. | +| `variants` | `fn variants(&self) -> Vec>` | All variants in this model. | +| `select_variant` | `fn select_variant(&self, variant: &Model) -> Result<(), FoundryLocalError>` | Select a variant from `variants()`. | +| `select_variant_by_id` | `fn select_variant_by_id(&self, id: &str) -> Result<(), FoundryLocalError>` | Select a variant by its unique id string. | +| `is_cached` | `async fn is_cached(&self) -> Result` | Whether the selected variant is cached on disk. | +| `is_loaded` | `async fn is_loaded(&self) -> Result` | Whether the selected variant is loaded in memory. | +| `download` | `async fn download(&self, progress: Option) -> Result<(), FoundryLocalError>` | Download the selected variant. `F: FnMut(f64) + Send + 'static` — receives progress as a percentage (0.0–100.0). | +| `path` | `async fn path(&self) -> Result` | Local file-system path of the selected variant. | +| `load` | `async fn load(&self) -> Result<(), FoundryLocalError>` | Load the selected variant into memory. | +| `unload` | `async fn unload(&self) -> Result<(), FoundryLocalError>` | Unload the selected variant from memory. | +| `remove_from_cache` | `async fn remove_from_cache(&self) -> Result<(), FoundryLocalError>` | Remove the selected variant from the local cache. | +| `create_chat_client` | `fn create_chat_client(&self) -> ChatClient` | **Deprecated** — use `ChatSession::new(&model)`. Create a ChatClient bound to the selected variant. | +| `create_audio_client` | `fn create_audio_client(&self) -> AudioClient` | **Deprecated** — use `AudioSession::new(&model)`. Create an AudioClient bound to the selected variant. | + +--- + +## OpenAI Clients + +> **Deprecated.** The OpenAI direct clients (`ChatClient`, `EmbeddingClient`, `AudioClient`, +> `LiveAudioTranscriptionSession`) are deprecated in favor of the Session API +> ([`ChatSession`](#chatsession), [`EmbeddingsSession`](#embeddingssession), +> [`AudioSession`](#audiosession)). They remain available for backward compatibility. + +### ChatClient + +OpenAI-compatible chat completions backed by a local model. Uses a consuming builder pattern. + +```rust +pub struct ChatClient { /* private fields */ } +``` + +**Builder methods** (all `mut self -> Self`): + +| Method | Signature | Description | +|--------|-----------|-------------| +| `frequency_penalty` | `fn frequency_penalty(mut self, v: f64) -> Self` | Set the frequency penalty. | +| `max_tokens` | `fn max_tokens(mut self, v: u32) -> Self` | Maximum tokens to generate. | +| `n` | `fn n(mut self, v: u32) -> Self` | Number of completions. | +| `temperature` | `fn temperature(mut self, v: f64) -> Self` | Sampling temperature. | +| `presence_penalty` | `fn presence_penalty(mut self, v: f64) -> Self` | Presence penalty. | +| `top_p` | `fn top_p(mut self, v: f64) -> Self` | Nucleus sampling probability. | +| `top_k` | `fn top_k(mut self, v: u32) -> Self` | Top-k sampling *(Foundry extension)*. | +| `random_seed` | `fn random_seed(mut self, v: u64) -> Self` | Random seed for reproducibility *(Foundry extension)*. | +| `response_format` | `fn response_format(mut self, v: ChatResponseFormat) -> Self` | Desired response format. | +| `tool_choice` | `fn tool_choice(mut self, v: ChatToolChoice) -> Self` | Tool choice strategy. | + +**Completion methods:** + +| Method | Signature | Description | +|--------|-----------|-------------| +| `complete_chat` | `async fn complete_chat(&self, messages: &[ChatCompletionRequestMessage], tools: Option<&[ChatCompletionTools]>) -> Result` | Non-streaming chat completion. | +| `complete_streaming_chat` | `async fn complete_streaming_chat(&self, messages: &[ChatCompletionRequestMessage], tools: Option<&[ChatCompletionTools]>) -> Result` | Streaming chat completion. | + +**Example:** +```rust +let client = model.create_chat_client() + .temperature(0.7) + .max_tokens(256); +``` + +--- + +### ChatCompletionStream + +```rust +pub type ChatCompletionStream = JsonStream; +``` + +A stream of `CreateChatCompletionStreamResponse` chunks. Use with `StreamExt::next()`. + +--- + +### EmbeddingClient + +OpenAI-compatible embedding generation backed by a local model. + +| Method | Description | +|---|---| +| `new(model_id, core)` | *(internal)* Create a new client | +| `generate_embedding(input: &str) -> Result` | Generate embedding for a single input | +| `generate_embeddings(inputs: &[&str]) -> Result` | Generate embeddings for multiple inputs | + +Returns `async_openai::types::embeddings::CreateEmbeddingResponse`: + +| Field | Type | Description | +|---|---|---| +| `model` | `String` | Model used for generation | +| `object` | `String` | Object type (always `"list"`) | +| `data` | `Vec` | List of embedding results | +| `usage` | `Usage` | Token usage information | + +Each `Embedding` in `data`: + +| Field | Type | Description | +|---|---|---| +| `index` | `u32` | Index of this embedding in the batch | +| `embedding` | `Vec` | The embedding vector (float32) | + +--- + +### AudioClient + +OpenAI-compatible audio transcription backed by a local model. + +```rust +pub struct AudioClient { /* private fields */ } +``` + +**Builder methods** (all `mut self -> Self`): + +| Method | Signature | Description | +|--------|-----------|-------------| +| `language` | `fn language(mut self, lang: impl Into) -> Self` | Language hint for transcription. | +| `temperature` | `fn temperature(mut self, v: f64) -> Self` | Sampling temperature. | + +**Transcription methods:** + +| Method | Signature | Description | +|--------|-----------|-------------| +| `transcribe` | `async fn transcribe(&self, audio_file_path: impl AsRef) -> Result` | Transcribe an audio file. | +| `transcribe_streaming` | `async fn transcribe_streaming(&self, audio_file_path: impl AsRef) -> Result` | Streaming transcription. | + +**Example:** +```rust +let client = model.create_audio_client() + .language("en") + .temperature(0.2); +``` + +--- + +### AudioTranscriptionStream + +```rust +pub type AudioTranscriptionStream = JsonStream; +``` + +A stream of `AudioTranscriptionResponse` chunks. Use with `StreamExt::next()`. + +--- + +### AudioTranscriptionResponse + +```rust +pub struct AudioTranscriptionResponse { + pub text: String, // The transcribed text + pub language: Option, // Language of input audio (if detected) + pub duration: Option, // Duration in seconds (if available) + pub segments: Option>, // Transcription segments (if available) + pub words: Option>, // Words with timestamps (if available) +} +``` + +Derives: `Debug`, `Clone`, `Deserialize`, `Serialize` + +--- + +### TranscriptionSegment + +A segment of a transcription, as returned by the OpenAI-compatible API. + +```rust +pub struct TranscriptionSegment { + pub id: i32, + pub seek: i32, + pub start: f64, + pub end: f64, + pub text: String, + pub tokens: Option>, + pub temperature: Option, + pub avg_logprob: Option, + pub compression_ratio: Option, + pub no_speech_prob: Option, +} +``` + +Derives: `Debug`, `Clone`, `Deserialize`, `Serialize` + +--- + +### TranscriptionWord + +A word with timing information, as returned by the OpenAI-compatible API. + +```rust +pub struct TranscriptionWord { + pub word: String, + pub start: f64, + pub end: f64, +} +``` + +Derives: `Debug`, `Clone`, `Deserialize`, `Serialize` + +--- + +### JsonStream\ + +Generic stream that deserializes each received JSON string chunk into `T`. Empty chunks are silently skipped. + +```rust +pub struct JsonStream { /* private fields */ } + +impl Unpin for JsonStream {} +impl Stream for JsonStream { + type Item = Result; +} +``` + +--- + +## Inference API + +The low-level, modality-agnostic inference API, mirroring the C++/C#/Python/JS SDKs +but expressed idiomatically in Rust. Submit a [`Request`](#request) of +[`Item`](#item)s to a [`Session`](#session) and receive a [`Response`](#response) +of `Item`s. + +The design splits types by whether they own a native lifetime: + +- **Pure-data values** — `Item`, `Request`, `Response`, and their option/enum + types are plain owned data: `Send + Sync + Clone`, holding no native handle. + Build them freely, move them across threads, and drop them at will; the native + layer copies whatever it needs when a request is processed. +- **Handle-backed** — `Session` (and the typed `ChatSession` / + `EmbeddingsSession` / `AudioSession`) and `ItemQueue` wrap a shared native + object (`Arc`), so cloning yields another handle to the same underlying + session/queue. + +### Session + +A stateful inference session bound to a loaded [`Model`](#model). The base, +modality-agnostic entry point. + +```rust +pub struct Session { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `async fn new(model: &Model) -> Result` | Open a session on a loaded model. | +| `process_request` | `async fn process_request(&self, request: Request) -> Result` | Process a request and return the complete response. Cancel-on-drop: dropping the future (e.g. via `tokio::time::timeout` or `select!`) cancels the in-flight request. | +| `process_streaming_request` | `fn process_streaming_request(&self, request: Request) -> ItemStream` | Process a request, streaming each output item as it is produced. Dropping the stream cancels generation. | +| `set_options` | `async fn set_options(&self, options: RequestOptions) -> Result<(), FoundryLocalError>` | Apply session-scoped options that persist across subsequent requests. | +| `create_input_queue` | `fn create_input_queue(&self) -> Result` | Create an [`ItemQueue`](#itemqueue) for streaming incremental input into a request. | + +### ChatSession + +A chat-oriented session adding tool registration and turn management. +Dereferences to [`Session`](#session), so all base methods are available. + +```rust +pub struct ChatSession { /* private fields */ } +impl Deref for ChatSession { type Target = Session; } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `async fn new(model: &Model) -> Result` | Open a chat session on a loaded model. | +| `add_tool_definition` | `async fn add_tool_definition(&self, definition: ToolDefinition) -> Result<(), FoundryLocalError>` | Register a tool for the lifetime of the session. | +| `remove_tool_definition` | `async fn remove_tool_definition(&self, name: impl Into) -> Result` | Remove a tool by name; returns whether one was removed. | +| `turn_count` | `fn turn_count(&self) -> usize` | The number of completed conversation turns. | +| `undo_turns` | `async fn undo_turns(&self, count: usize) -> Result<(), FoundryLocalError>` | Rewind the last `count` turns. | +| `into_session` | `fn into_session(self) -> Session` | Consume this handle, yielding the base session. | + +### EmbeddingsSession + +An embeddings-oriented session producing dense vectors for text input. +Dereferences to [`Session`](#session). + +```rust +pub struct EmbeddingsSession { /* private fields */ } +impl Deref for EmbeddingsSession { type Target = Session; } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `async fn new(model: &Model) -> Result` | Open an embeddings session on a loaded model. | +| `embed` | `async fn embed(&self, input: impl Into) -> Result, FoundryLocalError>` | Embed a single text input. | +| `embed_batch` | `async fn embed_batch(&self, inputs: Vec) -> Result>, FoundryLocalError>` | Embed a batch of inputs, one vector per input (in order). | +| `into_session` | `fn into_session(self) -> Session` | Consume this handle, yielding the base session. | + +### AudioSession + +An audio-oriented session for speech tasks (e.g. transcription). +Dereferences to [`Session`](#session). + +```rust +pub struct AudioSession { /* private fields */ } +impl Deref for AudioSession { type Target = Session; } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `async fn new(model: &Model) -> Result` | Open an audio session on a loaded model. | +| `transcribe` | `async fn transcribe(&self, audio: Item) -> Result` | Transcribe a single audio item, returning the recognized text. | +| `into_session` | `fn into_session(self) -> Session` | Consume this handle, yielding the base session. | + +### ItemQueue + +A thread-safe, multi-producer / multi-consumer queue of [`Item`](#item)s for +streaming incremental input into a request (e.g. live audio). Create one from a +session via [`Session::create_input_queue`](#session) and attach it to a request +with [`Request::with_input_queue`](#request). Cloning yields another handle to the +same underlying queue. + +```rust +pub struct ItemQueue { /* private fields */ } +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `push` | `fn push(&self, item: &Item) -> Result<(), FoundryLocalError>` | Push an item, transferring a native copy into the queue. | +| `try_pop` | `fn try_pop(&self) -> Option` | Pop the next item, or `None` if currently empty. | +| `len` | `fn len(&self) -> usize` | Number of items currently buffered. | +| `is_empty` | `fn is_empty(&self) -> bool` | Whether the queue currently holds no items. | +| `mark_finished` | `fn mark_finished(&self)` | Signal that no more items will be pushed. | +| `is_finished` | `fn is_finished(&self) -> bool` | Whether `mark_finished` has been called. | + +### ItemStream + +An asynchronous stream of output [`Item`](#item)s from +[`Session::process_streaming_request`](#session). Implements +[`futures_core::Stream`]; dropping it cancels generation. + +```rust +pub struct ItemStream { /* private fields */ } + +impl Unpin for ItemStream {} +impl Stream for ItemStream { + type Item = Result; +} +``` + +### Item + +A single unit of input or output data exchanged with a session. A pure-data, +owned enum (`Send + Sync + Clone`) — construct with the associated functions and +inspect by pattern matching. + +```rust +pub enum Item { + Text { text: String, kind: TextKind }, + Message(Message), + Bytes(Vec), + Tensor(Tensor), + Image(Image), + Audio(Audio), + ToolCall(ToolCall), + ToolResult(ToolResult), + SpeechSegment(SpeechSegment), // output-only + SpeechResult(SpeechResult), // output-only +} +``` + +**Constructors:** + +| Method | Signature | Description | +|--------|-----------|-------------| +| `text` | `fn text(text: impl Into) -> Item` | A `Default`-kind text item. | +| `reasoning` | `fn reasoning(text: impl Into) -> Item` | A `Reasoning`-kind text item. | +| `message` | `fn message(role: MessageRole, content: impl Into>) -> Item` | A chat message with content parts. | +| `system_message` / `user_message` / `assistant_message` / `developer_message` / `tool_message` | `fn(content: impl Into>) -> Item` | Role-specific message constructors. | +| `bytes` | `fn bytes(data: impl Into>) -> Item` | An opaque byte buffer. | +| `tensor` | `fn tensor(data_type: TensorDataType, shape: impl Into>, data: impl Into>) -> Item` | A numeric tensor from raw bytes. | +| `float_tensor` | `fn float_tensor(shape: impl Into>, data: &[f32]) -> Item` | A `Float`-typed tensor from `f32` values. | +| `image_data` / `image_uri` | `fn(…, format: Option>) -> Item` | An inline or URI-referenced image. | +| `audio_data` / `audio_uri` | `fn(…) -> Item` | An inline or URI-referenced audio clip. | +| `tool_call` | `fn tool_call(call_id: impl Into, name: impl Into, arguments: impl Into) -> Item` | A model-issued tool call. | +| `tool_result` | `fn tool_result(call_id: impl Into, result: impl Into) -> Item` | The result of executing a tool call. | + +**Accessors:** + +| Method | Signature | Description | +|--------|-----------|-------------| +| `item_type` | `fn item_type(&self) -> ItemType` | The discriminant of this item. | +| `as_text` | `fn as_text(&self) -> Option<&str>` | The text, if this is a `Text` item. | +| `as_message` | `fn as_message(&self) -> Option<&Message>` | The message, if this is a `Message` item. | +| `as_tensor` | `fn as_tensor(&self) -> Option<&Tensor>` | The tensor, if this is a `Tensor` item. | +| `as_tool_call` | `fn as_tool_call(&self) -> Option<&ToolCall>` | The tool call, if this is a `ToolCall` item. | +| `as_speech_result` | `fn as_speech_result(&self) -> Option<&SpeechResult>` | The speech result, if this is a `SpeechResult` item. | + +Supporting enums: `ItemType`, `TextKind` (`Default`, `Reasoning`, `OpenAiJson`), +`MessageRole` (`None`, `System`, `User`, `Assistant`, `Tool`, `Developer`), +`TensorDataType`, `SpeechSegmentKind`, and `MediaSource` (`Data(Vec)` / +`Uri(String)`). + +### Message + +A chat message with a role and nested content parts. + +```rust +pub struct Message { + pub role: MessageRole, + pub content: Vec, + pub name: Option, +} +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `fn new(role: MessageRole, content: impl Into>) -> Message` | Construct a message. | +| `with_name` | `fn with_name(mut self, name: impl Into) -> Message` | Attach a participant name (builder-style). | +| `is_simple_text` | `fn is_simple_text(&self) -> bool` | Whether the content is a single text part. | +| `text` | `fn text(&self) -> String` | The concatenated text of all text content parts. | + +### Tensor + +A numeric tensor payload (e.g. an embedding vector). + +```rust +pub struct Tensor { + pub data_type: TensorDataType, + pub shape: Vec, + pub data: Vec, +} +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `as_f32` | `fn as_f32(&self) -> Option>` | Reinterpret the bytes as `f32` values, if `data_type` is `Float`. | + +### Request + +A unit of work submitted to a session. Pure data: owned input items, an optional +streaming [`ItemQueue`](#itemqueue), and optional [`RequestOptions`](#requestoptions). + +```rust +pub struct Request { + pub items: Vec, + pub input_queue: Option, + pub options: Option, +} +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `fn new() -> Request` | An empty request. | +| `from_items` | `fn from_items(items: impl Into>) -> Request` | A request from a list of input items. | +| `with_item` | `fn with_item(mut self, item: Item) -> Request` | Append an input item (builder-style). | +| `with_input_queue` | `fn with_input_queue(mut self, queue: ItemQueue) -> Request` | Attach a streaming input queue (builder-style). | +| `with_options` | `fn with_options(mut self, options: RequestOptions) -> Request` | Attach per-request options (builder-style). | + +### RequestOptions + +Sampling / decoding parameters applied to a request (or, via +[`Session::set_options`](#session), to a session). Typed `search` fields and +`tool_choice` take precedence over `additional_options` on key collision. + +```rust +pub struct RequestOptions { + pub search: SearchOptions, + pub tool_choice: Option, + pub additional_options: Vec<(String, String)>, +} + +pub struct SearchOptions { + pub temperature: Option, + pub top_p: Option, + pub top_k: Option, + pub max_output_tokens: Option, + pub frequency_penalty: Option, + pub presence_penalty: Option, + pub seed: Option, + pub early_stopping: Option, + pub do_sample: Option, +} + +pub enum ToolChoice { Auto, None, Required } +``` + +### Response + +The result of processing a request: output items plus the finish reason and token +usage. + +```rust +pub struct Response { + pub items: Vec, + pub finish_reason: FinishReason, + pub usage: Usage, +} + +pub struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, +} +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `text` | `fn text(&self) -> String` | The concatenated text of all text output items. | + +### FinishReason + +Why generation stopped. (Distinct from the OpenAI facade's finish reason, which is +re-exported as [`ChatFinishReason`](#re-exported-openai-types).) + +```rust +pub enum FinishReason { None, Error, Stop, Length, ToolCalls } +``` + +### ToolDefinition + +A tool the model may call, registered on a [`ChatSession`](#chatsession). + +```rust +pub struct ToolDefinition { + pub name: String, + pub description: Option, + pub json_schema: String, +} +``` + +| Method | Signature | Description | +|--------|-----------|-------------| +| `new` | `fn new(name: impl Into, json_schema: impl Into) -> ToolDefinition` | A tool with a name and JSON-schema parameters. | +| `with_description` | `fn with_description(mut self, description: impl Into) -> ToolDefinition` | Attach a description (builder-style). | + +--- + +## Types + +### ModelInfo + +Full metadata for a model variant as returned by the catalog. + +```rust +pub struct ModelInfo { + pub id: String, + pub name: String, + pub version: u64, + pub alias: String, + pub display_name: Option, + pub provider_type: String, + pub uri: String, + pub model_type: String, + pub prompt_template: Option, + pub publisher: Option, + pub model_settings: Option, + pub license: Option, + pub license_description: Option, + pub cached: bool, + pub task: Option, + pub runtime: Option, + pub file_size_mb: Option, + pub supports_tool_calling: Option, + pub max_output_tokens: Option, + pub min_fl_version: Option, + pub created_at_unix: u64, +} +``` + +Derives: `Debug`, `Clone`, `Deserialize` + +--- + +### ChatResponseFormat + +```rust +pub enum ChatResponseFormat { + Text, // Plain text output (default) + JsonObject, // JSON output (unstructured) + JsonSchema(String), // JSON constrained by schema string + LarkGrammar(String), // Lark grammar constraint (Foundry extension) +} +``` + +--- + +### ChatToolChoice + +```rust +pub enum ChatToolChoice { + None, // Model will not call any tool + Auto, // Model decides whether to call a tool + Required, // Model must call at least one tool + Function(String), // Model must call the named function +} +``` + +--- + +### DeviceType + +```rust +pub enum DeviceType { + Invalid, + CPU, + GPU, + NPU, +} +``` + +--- + +### PromptTemplate + +```rust +pub struct PromptTemplate { + pub system: Option, + pub user: Option, + pub assistant: Option, + pub prompt: Option, +} +``` + +--- + +### Runtime + +```rust +pub struct Runtime { + pub device_type: DeviceType, + pub execution_provider: String, +} +``` + +--- + +### ModelSettings + +```rust +pub struct ModelSettings { + pub parameters: Option>, +} +``` + +--- + +### Parameter + +```rust +pub struct Parameter { + pub name: String, + pub value: Option, +} +``` + +--- + +## Error Handling + +### FoundryLocalError + +```rust +pub enum FoundryLocalError { + /// The native core library could not be loaded. + LibraryLoad { reason: String }, + + /// A command executed against the native core returned an error. + CommandExecution { reason: String }, + + /// The provided configuration is invalid. + InvalidConfiguration { reason: String }, + + /// A model operation failed (load, unload, download, etc.). + ModelOperation { reason: String }, + + /// An HTTP request to the external service failed. + HttpRequest(reqwest::Error), + + /// Serialization or deserialization of JSON data failed. + Serialization(serde_json::Error), + + /// A validation check on user-supplied input failed. + Validation { reason: String }, + + /// An I/O error occurred. + Io(std::io::Error), + + /// An internal SDK error (e.g. poisoned lock). + Internal { reason: String }, +} +``` + +Implements: `Display`, `Error`, `From`, `From`, `From` + +> **Note:** The `Result` type alias (`std::result::Result`) is defined +> in `error.rs` for internal SDK use but is **not** re-exported from the crate root. +> Public API signatures use `Result` explicitly to avoid shadowing +> the standard `Result`. + +--- + +## Re-exported OpenAI Types + +The following types from `async_openai` are re-exported at the crate root for convenience: + +**Request types:** +- `ChatCompletionRequestMessage` +- `ChatCompletionRequestSystemMessage` +- `ChatCompletionRequestUserMessage` +- `ChatCompletionRequestAssistantMessage` +- `ChatCompletionRequestToolMessage` +- `ChatCompletionTools` +- `ChatCompletionToolChoiceOption` +- `ChatCompletionNamedToolChoice` +- `FunctionObject` + +**Response types:** +- `CreateChatCompletionResponse` +- `CreateChatCompletionStreamResponse` +- `ChatChoice` +- `ChatChoiceStream` +- `ChatCompletionResponseMessage` +- `ChatCompletionStreamResponseDelta` +- `CompletionUsage` +- `ChatFinishReason` (OpenAI finish reason; re-exported under this name to avoid + colliding with the core inference [`FinishReason`](#finishreason)) + +**Tool call types:** +- `ChatCompletionMessageToolCall` +- `ChatCompletionMessageToolCallChunk` +- `ChatCompletionMessageToolCalls` +- `FunctionCall` +- `FunctionCallStream` diff --git a/sdk_v2/rust/examples/chat_completion.rs b/sdk_v2/rust/examples/chat_completion.rs new file mode 100644 index 000000000..ce875d06a --- /dev/null +++ b/sdk_v2/rust/examples/chat_completion.rs @@ -0,0 +1,101 @@ +//! Basic chat completion example demonstrating synchronous and streaming +//! usage of the Foundry Local SDK's `Session` API. + +use std::io::{self, Write}; + +use foundry_local_sdk::{ + ChatSession, FoundryLocalConfig, FoundryLocalError, FoundryLocalManager, Item, Request, + RequestOptions, SearchOptions, +}; +use tokio_stream::StreamExt; + +/// Convenience alias matching the SDK's internal Result type. +type Result = std::result::Result; + +#[tokio::main] +async fn main() -> Result<()> { + // ── 1. Initialise the manager ──────────────────────────────────────── + let config = FoundryLocalConfig::new("foundry_local_samples"); + + let manager = FoundryLocalManager::create(config)?; + + // ── 2. List available models ───────────────────────────────────────── + let models = manager.catalog().get_models().await?; + println!("Available models:"); + for model in &models { + println!(" • {} (id: {})", model.alias(), model.id()); + } + + // ── 3. Pick a model and ensure it is loaded ────────────────────────── + // Prefer a known chat model; fall back to the first available. + let model_alias = ["phi-3.5-mini", "phi-4-mini"] + .iter() + .find(|alias| models.iter().any(|m| m.alias() == **alias)) + .map(|s| s.to_string()) + .or_else(|| models.first().map(|m| m.alias().to_string())) + .expect("No models available in the catalog"); + + let model = manager.catalog().get_model(&model_alias).await?; + + if !model.is_cached().await? { + println!("Downloading model '{}'…", model.alias()); + model + .download(Some(|progress: f64| { + println!(" {progress:.1}%"); + })) + .await?; + } + + println!("Loading model '{}'…", model.alias()); + model.load().await?; + + // ── 4. Open a chat session ─────────────────────────────────────────── + // `ChatSession` retains conversation state across turns; options set here + // persist, so each request only needs to carry the new input. + let session = ChatSession::new(&model).await?; + session + .set_options(RequestOptions { + search: SearchOptions { + temperature: Some(0.7), + max_output_tokens: Some(256), + ..Default::default() + }, + ..Default::default() + }) + .await?; + + // ── 5. Synchronous request ─────────────────────────────────────────── + let request = Request::from_items(vec![ + Item::system_message(vec![Item::text("You are a helpful assistant.")]), + Item::user_message(vec![Item::text("What is Rust's ownership model?")]), + ]); + + println!("\n--- Synchronous completion ---"); + let response = session.process_request(request).await?; + println!("Assistant: {}", response.text()); + + // ── 6. Streaming request (continues the same conversation) ─────────── + // The session already holds the system prompt and the previous turn, so we + // only send the new user message here. + println!("\n--- Streaming completion ---"); + let streaming = Request::from_items(vec![Item::user_message(vec![Item::text( + "Explain the borrow checker in two sentences.", + )])]); + + print!("Assistant: "); + let mut stream = session.process_streaming_request(streaming); + while let Some(item) = stream.next().await { + if let Some(text) = item?.as_text() { + print!("{text}"); + io::stdout().flush().ok(); + } + } + println!(); + + // ── 7. Unload the model────────────────────────────────────────────── + println!("\nUnloading model…"); + model.unload().await?; + println!("Done."); + + Ok(()) +} diff --git a/sdk_v2/rust/examples/interactive_chat.rs b/sdk_v2/rust/examples/interactive_chat.rs new file mode 100644 index 000000000..c048d4c3f --- /dev/null +++ b/sdk_v2/rust/examples/interactive_chat.rs @@ -0,0 +1,113 @@ +//! Interactive chat example — a simple terminal chatbot powered by Foundry Local. +//! +//! Run with: `cargo run --example interactive_chat` + +use std::io::{self, Write}; + +use foundry_local_sdk::{ + ChatSession, FoundryLocalConfig, FoundryLocalManager, Item, Request, RequestOptions, + SearchOptions, +}; +use tokio_stream::StreamExt; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // ── Initialise ─────────────────────────────────────────────────────── + let manager = FoundryLocalManager::create(FoundryLocalConfig::new("foundry_local_samples"))?; + + // Pick the first available model (or change this to a specific alias) + let catalog = manager.catalog(); + let models = catalog.get_models().await?; + + println!("Available models:"); + for (i, m) in models.iter().enumerate() { + println!(" [{i}] {}", m.alias()); + } + + print!("\nSelect a model number (default 0): "); + io::stdout().flush()?; + let mut choice = String::new(); + io::stdin().read_line(&mut choice)?; + let idx: usize = choice.trim().parse().unwrap_or(0); + + let alias = models + .get(idx) + .map(|m| m.alias().to_string()) + .unwrap_or_else(|| models[0].alias().to_string()); + + let model = catalog.get_model(&alias).await?; + + // Download if needed + if !model.is_cached().await? { + println!("Downloading '{alias}'…"); + model.download(Some(|p: f64| print!("\r {p:.1}%"))).await?; + println!(); + } + + println!("Loading '{alias}'…"); + model.load().await?; + println!("Ready! Type your messages below. Press Ctrl-D (or type 'quit') to exit.\n"); + + // ── Chat loop ──────────────────────────────────────────────────────── + // `ChatSession` retains conversation history internally, so each turn only + // needs to send the new user message — no manual message bookkeeping. + let session = ChatSession::new(&model).await?; + session + .set_options(RequestOptions { + search: SearchOptions { + temperature: Some(0.7), + max_output_tokens: Some(512), + ..Default::default() + }, + ..Default::default() + }) + .await?; + + loop { + print!("You: "); + io::stdout().flush()?; + + let mut input = String::new(); + if io::stdin().read_line(&mut input)? == 0 { + break; // EOF (Ctrl-D) + } + + let input = input.trim(); + if input.is_empty() { + continue; + } + if input.eq_ignore_ascii_case("quit") || input.eq_ignore_ascii_case("exit") { + break; + } + + // Seed the system prompt on the first turn only; the session keeps it. + let mut items = Vec::new(); + if session.turn_count() == 0 { + items.push(Item::system_message(vec![Item::text( + "You are a helpful, concise assistant.", + )])); + } + items.push(Item::user_message(vec![Item::text(input)])); + + // Stream the response token by token. The session records the turn, so + // there is no need to append the reply to a local history buffer. + print!("Assistant: "); + io::stdout().flush()?; + + let mut stream = session.process_streaming_request(Request::from_items(items)); + while let Some(item) = stream.next().await { + if let Some(text) = item?.as_text() { + print!("{text}"); + io::stdout().flush().ok(); + } + } + println!("\n"); + } + + // ── Cleanup ────────────────────────────────────────────────────────── + println!("\nUnloading model…"); + model.unload().await?; + println!("Goodbye!"); + + Ok(()) +} diff --git a/sdk_v2/rust/examples/low_level_inference.rs b/sdk_v2/rust/examples/low_level_inference.rs new file mode 100644 index 000000000..45b43914a --- /dev/null +++ b/sdk_v2/rust/examples/low_level_inference.rs @@ -0,0 +1,85 @@ +//! Low-level inference example using the modality-agnostic `Session` / `Item` / +//! `Request` / `Response` API (as opposed to the higher-level OpenAI facade). +//! +//! Demonstrates a synchronous request, a streaming request, and multi-turn chat +//! with `ChatSession`. + +use std::io::{self, Write}; + +use foundry_local_sdk::{ + ChatSession, FoundryLocalConfig, FoundryLocalError, FoundryLocalManager, Item, MessageRole, + Request, +}; +use tokio_stream::StreamExt; + +/// Convenience alias matching the SDK's internal Result type. +type Result = std::result::Result; + +#[tokio::main] +async fn main() -> Result<()> { + // ── 1. Initialise the manager and load a chat model ────────────────── + let manager = FoundryLocalManager::create(FoundryLocalConfig::new("foundry_local_samples"))?; + + let models = manager.catalog().get_models().await?; + let model_alias = ["phi-3.5-mini", "phi-4-mini"] + .iter() + .find(|alias| models.iter().any(|m| m.alias() == **alias)) + .map(|s| s.to_string()) + .or_else(|| models.first().map(|m| m.alias().to_string())) + .expect("No models available in the catalog"); + + let model = manager.catalog().get_model(&model_alias).await?; + if !model.is_cached().await? { + println!("Downloading model '{}'…", model.alias()); + model.download(None::).await?; + } + println!("Loading model '{}'…", model.alias()); + model.load().await?; + + // ── 2. Open a chat session ─────────────────────────────────────────── + // `ChatSession` derefs to `Session`, so `process_request` / + // `process_streaming_request` are available directly, plus turn management. + let session = ChatSession::new(&model).await?; + + // ── 3. Synchronous request ─────────────────────────────────────────── + // A `Request` is pure data: a list of `Item`s. Here a single user message. + let request = Request::from_items(vec![Item::message( + MessageRole::User, + vec![Item::text("What is Rust's ownership model?")], + )]); + + println!("\n--- Synchronous request ---"); + let response = session.process_request(request).await?; + println!("Assistant: {}", response.text()); + println!( + "(finish: {:?}, tokens: {} prompt + {} completion)", + response.finish_reason, response.usage.prompt_tokens, response.usage.completion_tokens + ); + + // ── 4. Streaming request ───────────────────────────────────────────── + // `process_streaming_request` yields each output `Item` as it is produced. + println!("\n--- Streaming request ---"); + let streaming = Request::from_items(vec![Item::user_message(vec![Item::text( + "Explain the borrow checker in two sentences.", + )])]); + + print!("Assistant: "); + let mut stream = session.process_streaming_request(streaming); + while let Some(item) = stream.next().await { + if let Some(text) = item?.as_text() { + print!("{text}"); + io::stdout().flush().ok(); + } + } + println!(); + + // ── 5. Multi-turn state ────────────────────────────────────────────── + // The session retains conversation state; each processed request is a turn. + println!("\nCompleted {} turn(s).", session.turn_count()); + + // ── 6. Unload the model ────────────────────────────────────────────── + model.unload().await?; + println!("Done."); + + Ok(()) +} diff --git a/sdk_v2/rust/examples/tool_calling.rs b/sdk_v2/rust/examples/tool_calling.rs new file mode 100644 index 000000000..b058b1a78 --- /dev/null +++ b/sdk_v2/rust/examples/tool_calling.rs @@ -0,0 +1,148 @@ +//! Tool-calling example demonstrating how to define tools with the `Session` +//! API, handle streamed `Item::ToolCall`s, execute the tool locally, and feed +//! results back for a multi-turn conversation. + +use std::io::{self, Write}; + +use serde_json::{json, Value}; +use tokio_stream::StreamExt; + +use foundry_local_sdk::{ + ChatSession, FoundryLocalConfig, FoundryLocalError, FoundryLocalManager, Item, Request, + RequestOptions, SearchOptions, ToolChoice, ToolDefinition, +}; + +/// Convenience alias matching the SDK's internal Result type. +type Result = std::result::Result; + +/// A trivial tool that multiplies two numbers. +fn multiply(a: f64, b: f64) -> f64 { + a * b +} + +/// Dispatch a tool call by name and arguments. +fn invoke_tool(name: &str, arguments: &Value) -> Result { + match name { + "multiply" => { + let a = arguments.get("a").and_then(|v| v.as_f64()).unwrap_or(0.0); + let b = arguments.get("b").and_then(|v| v.as_f64()).unwrap_or(0.0); + let result = multiply(a, b); + Ok(result.to_string()) + } + _ => Ok(format!("Unknown tool: {name}")), + } +} + +#[tokio::main] +async fn main() -> Result<()> { + // ── 1. Initialise ──────────────────────────────────────────────────── + let config = FoundryLocalConfig::new("foundry_local_samples"); + + let manager = FoundryLocalManager::create(config)?; + + // ── 2. Load a model ────────────────────────────────────────────────── + let models = manager.catalog().get_models().await?; + let model = models + .iter() + .find(|m| m.info().supports_tool_calling == Some(true)) + .or_else(|| models.first()) + .expect("No models available"); + + if !model.is_cached().await? { + println!("Downloading model '{}'…", model.alias()); + model.download(Some(|p: f64| println!(" {p:.1}%"))).await?; + } + println!("Loading model '{}'…", model.alias()); + model.load().await?; + + // ── 3. Open a chat session and register the tool ───────────────────── + // Tools are registered once and remain available for every request on the + // session. `json_schema` describes just the tool's parameters. + let session = ChatSession::new(model).await?; + let multiply_params = json!({ + "type": "object", + "properties": { + "a": { "type": "number", "description": "First operand" }, + "b": { "type": "number", "description": "Second operand" } + }, + "required": ["a", "b"] + }); + session + .add_tool_definition( + ToolDefinition::new("multiply", multiply_params.to_string()) + .with_description("Multiply two numbers together."), + ) + .await?; + + // ── 4. First turn: force a tool call ───────────────────────────────── + // `tool_choice = Required` makes the model call a tool. Streamed items + // arrive as complete `Item::ToolCall`s — no delta reassembly needed. + let request = Request::from_items(vec![ + Item::system_message(vec![Item::text( + "You are a helpful calculator assistant. Use the multiply tool when asked to multiply.", + )]), + Item::user_message(vec![Item::text("What is 6 times 7?")]), + ]) + .with_options(RequestOptions { + search: SearchOptions { + max_output_tokens: Some(512), + ..Default::default() + }, + tool_choice: Some(ToolChoice::Required), + ..Default::default() + }); + + println!("Sending initial request…"); + let mut tool_results = Vec::new(); + let mut stream = session.process_streaming_request(request); + while let Some(item) = stream.next().await { + let item = item?; + if let Some(call) = item.as_tool_call() { + let args: Value = serde_json::from_str(&call.arguments).unwrap_or_else(|_| json!({})); + println!("Tool call: {}({})", call.name, args); + let result = invoke_tool(&call.name, &args)?; + println!("Tool result: {result}"); + tool_results.push(Item::tool_result(call.call_id.clone(), result)); + } else if let Some(text) = item.as_text() { + print!("{text}"); + io::stdout().flush().ok(); + } + } + + if tool_results.is_empty() { + println!("(model did not request any tool calls)"); + model.unload().await?; + return Ok(()); + } + + // ── 5. Second turn: feed the results back for a final answer ───────── + // The session already retains the user question and the assistant's tool + // call, so only the tool results need to be sent. `tool_choice = None` + // asks the model to reply in natural language rather than call again. + let follow_up = Request::from_items(tool_results).with_options(RequestOptions { + search: SearchOptions { + max_output_tokens: Some(512), + ..Default::default() + }, + tool_choice: Some(ToolChoice::None), + ..Default::default() + }); + + println!("\nContinuing conversation…"); + print!("Assistant: "); + let mut stream = session.process_streaming_request(follow_up); + while let Some(item) = stream.next().await { + if let Some(text) = item?.as_text() { + print!("{text}"); + io::stdout().flush().ok(); + } + } + println!(); + + // ── 6. Clean up────────────────────────────────────────────────────── + println!("\nUnloading model…"); + model.unload().await?; + println!("Done."); + + Ok(()) +} diff --git a/sdk_v2/rust/src/catalog.rs b/sdk_v2/rust/src/catalog.rs new file mode 100644 index 000000000..73f3c93ee --- /dev/null +++ b/sdk_v2/rust/src/catalog.rs @@ -0,0 +1,158 @@ +//! Model catalog — discovery and lookup for available models. +//! +//! The native catalog (owned by the [`FoundryLocalManager`](crate::FoundryLocalManager)) +//! caches the model list and refreshes itself, so this is a thin async wrapper +//! that preserves the legacy public surface. + +use std::sync::Arc; + +use crate::detail::api::Api; +use crate::detail::manager::NativeManager; +use crate::detail::model::Model; +use crate::detail::native::NativeCatalog; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; + +/// The model catalog provides discovery and lookup for all available models. +pub struct Catalog { + native: NativeCatalog, + name: String, +} + +impl Catalog { + pub(crate) fn new( + api: Arc, + ptr: *mut crate::detail::ffi::flCatalog, + manager: Arc, + ) -> Result { + let native = NativeCatalog::new(api, ptr, manager); + let name = native.name().unwrap_or_else(|_| "default".into()); + Ok(Self { native, name }) + } + + /// Catalog name as reported by the native core. + pub fn name(&self) -> &str { + &self.name + } + + /// Refresh the catalog from the native core. + /// + /// **No-op.** The native catalog manages its own caching and refresh, so + /// there is nothing for the SDK to do here. This method is retained only for + /// API compatibility with the legacy SDK and always returns `Ok(())`. + pub async fn update_models(&self) -> Result<()> { + Ok(()) + } + + /// Return all known models keyed by alias. + pub async fn get_models(&self) -> Result>> { + let native = self.native.clone(); + spawn_blocking(move || { + native + .get_models()? + .into_iter() + .map(|m| Model::from_group(&native.api, m).map(Arc::new)) + .collect() + }) + .await + } + + /// Look up a model by its alias. + pub async fn get_model(&self, alias: &str) -> Result> { + if alias.trim().is_empty() { + return Err(FoundryLocalError::Validation { + reason: "Model alias must be a non-empty string".into(), + }); + } + let native = self.native.clone(); + let alias = alias.to_owned(); + spawn_blocking(move || match native.get_model(&alias)? { + Some(m) => Model::from_group(&native.api, m).map(Arc::new), + None => { + let available: Vec = native + .get_models() + .ok() + .map(|models| { + models + .iter() + .filter_map(|m| { + m.info_ptr().ok().map(|info| unsafe { + crate::detail::api::cstr_to_string((native + .api + .model_api() + .Info_GetAlias)( + info + )) + .unwrap_or_default() + }) + }) + .collect() + }) + .unwrap_or_default(); + Err(FoundryLocalError::ModelOperation { + reason: format!("Unknown model alias '{alias}'. Available: {available:?}"), + }) + } + }) + .await + } + + /// Look up a specific model variant by its unique id. + /// + /// NOTE: This will return a `Model` representing a single variant. Use + /// [`get_model`](Catalog::get_model) to obtain a `Model` with all + /// available variants. + pub async fn get_model_variant(&self, id: &str) -> Result> { + if id.trim().is_empty() { + return Err(FoundryLocalError::Validation { + reason: "Variant id must be a non-empty string".into(), + }); + } + let native = self.native.clone(); + let id = id.to_owned(); + spawn_blocking(move || match native.get_model_variant(&id)? { + Some(m) => Model::from_variant(&native.api, m).map(Arc::new), + None => Err(FoundryLocalError::ModelOperation { + reason: format!("Unknown variant id '{id}'."), + }), + }) + .await + } + + /// Return only the model variants that are currently cached on disk. + pub async fn get_cached_models(&self) -> Result>> { + let native = self.native.clone(); + spawn_blocking(move || { + native + .get_cached_models()? + .into_iter() + .map(|m| Model::from_variant(&native.api, m).map(Arc::new)) + .collect() + }) + .await + } + + /// Return model variants that are currently loaded into memory. + pub async fn get_loaded_models(&self) -> Result>> { + let native = self.native.clone(); + spawn_blocking(move || { + native + .get_loaded_models()? + .into_iter() + .map(|m| Model::from_variant(&native.api, m).map(Arc::new)) + .collect() + }) + .await + } + + /// Resolve the latest catalog version for the provided model or variant. + pub async fn get_latest_version(&self, model_or_model_variant: &Model) -> Result> { + let native = self.native.clone(); + let target = model_or_model_variant.selected_native().clone(); + spawn_blocking(move || { + let latest = native.get_latest_version(&target)?; + Model::from_variant(&native.api, latest).map(Arc::new) + }) + .await + } +} diff --git a/sdk_v2/rust/src/configuration.rs b/sdk_v2/rust/src/configuration.rs new file mode 100644 index 000000000..a826369b9 --- /dev/null +++ b/sdk_v2/rust/src/configuration.rs @@ -0,0 +1,262 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use crate::detail::api::{to_cstring, Api, Kvps}; +use crate::detail::ffi::*; +use crate::error::{FoundryLocalError, Result}; + +/// Log level for the Foundry Local service. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, + Fatal, +} + +impl LogLevel { + /// Map to the native `flLogLevel` value. + fn as_native(&self) -> flLogLevel { + match self { + Self::Trace => FOUNDRY_LOCAL_LOG_VERBOSE, + Self::Debug => FOUNDRY_LOCAL_LOG_DEBUG, + Self::Info => FOUNDRY_LOCAL_LOG_INFO, + Self::Warn => FOUNDRY_LOCAL_LOG_WARNING, + Self::Error => FOUNDRY_LOCAL_LOG_ERROR, + Self::Fatal => FOUNDRY_LOCAL_LOG_FATAL, + } + } +} + +/// Application-level logger that the SDK can use to emit diagnostic messages. +/// +/// This is a stub — the logger is stored in the configuration and passed +/// through to the manager, but it is not wired into the native core yet. +pub trait Logger: Send + Sync { + /// Log a message at the given severity level. + fn log(&self, level: LogLevel, message: &str); +} + +/// User-facing configuration for initializing the Foundry Local SDK. +/// +/// Construct with [`FoundryLocalConfig::new`] and customise via the builder +/// methods: +/// +/// ```ignore +/// let config = FoundryLocalConfig::new("my_app") +/// .log_level(LogLevel::Debug) +/// .model_cache_dir("/path/to/cache"); +/// ``` +#[derive(Default)] +pub struct FoundryLocalConfig { + app_name: String, + app_data_dir: Option, + model_cache_dir: Option, + logs_dir: Option, + log_level: Option, + web_service_urls: Option, + service_endpoint: Option, + library_path: Option, + additional_settings: Option>, + logger: Option>, +} + +impl fmt::Debug for FoundryLocalConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FoundryLocalConfig") + .field("app_name", &self.app_name) + .field("app_data_dir", &self.app_data_dir) + .field("model_cache_dir", &self.model_cache_dir) + .field("logs_dir", &self.logs_dir) + .field("log_level", &self.log_level) + .field("web_service_urls", &self.web_service_urls) + .field("service_endpoint", &self.service_endpoint) + .field("library_path", &self.library_path) + .field("additional_settings", &self.additional_settings) + .field("logger", &self.logger.as_ref().map(|_| "..")) + .finish() + } +} + +impl FoundryLocalConfig { + /// Create a new configuration with the given application name. + /// + /// All other fields default to `None`. Use the builder methods to + /// customise: + /// + /// ```ignore + /// let config = FoundryLocalConfig::new("my_app") + /// .log_level(LogLevel::Debug) + /// .model_cache_dir("/path/to/cache"); + /// ``` + pub fn new(app_name: impl Into) -> Self { + Self { + app_name: app_name.into(), + ..Self::default() + } + } + + /// Override the application-data directory. + pub fn app_data_dir(mut self, dir: impl Into) -> Self { + self.app_data_dir = Some(dir.into()); + self + } + + /// Override the model-cache directory. + pub fn model_cache_dir(mut self, dir: impl Into) -> Self { + self.model_cache_dir = Some(dir.into()); + self + } + + /// Override the logs directory. + pub fn logs_dir(mut self, dir: impl Into) -> Self { + self.logs_dir = Some(dir.into()); + self + } + + /// Set the log level. + pub fn log_level(mut self, level: LogLevel) -> Self { + self.log_level = Some(level); + self + } + + /// Set the web-service listen URLs (e.g. `"http://localhost:5273"`). + pub fn web_service_urls(mut self, urls: impl Into) -> Self { + self.web_service_urls = Some(urls.into()); + self + } + + /// Set an external service endpoint URL. + pub fn service_endpoint(mut self, endpoint: impl Into) -> Self { + self.service_endpoint = Some(endpoint.into()); + self + } + + /// Override the path to the native Foundry Local Core library. + pub fn library_path(mut self, path: impl Into) -> Self { + self.library_path = Some(path.into()); + self + } + + /// Add a single key-value pair to the additional settings map. + pub fn additional_setting(mut self, key: impl Into, value: impl Into) -> Self { + self.additional_settings + .get_or_insert_with(HashMap::new) + .insert(key.into(), value.into()); + self + } + + /// Provide an application logger. + /// + /// **Not wired to native logging.** The native core's configuration ABI + /// exposes only a log level and a logs directory (see [`log_level`] and + /// [`logs_dir`]) — there is no logger-callback hook — so a custom [`Logger`] + /// cannot currently receive the core's log records. The logger is stored for + /// forward compatibility and any SDK-side use, but setting it has no effect + /// on native logging today; use [`log_level`] / [`logs_dir`] to control core + /// logging. + /// + /// [`log_level`]: Self::log_level + /// [`logs_dir`]: Self::logs_dir + pub fn logger(mut self, logger: impl Logger + 'static) -> Self { + self.logger = Some(Box::new(logger)); + self + } + + // ── Crate-internal helpers ─────────────────────────────────────────────── + + /// The configured native library path override, if any. + pub(crate) fn library_path_ref(&self) -> Option<&str> { + self.library_path.as_deref() + } + + /// Take ownership of the configured logger (consumed once by the manager). + pub(crate) fn take_logger(&mut self) -> Option> { + self.logger.take() + } + + /// Build a native `flConfiguration` from this configuration. + /// + /// Returns [`FoundryLocalError::InvalidConfiguration`] when `app_name` is + /// empty or blank. + pub(crate) fn build_native(&self, api: &Arc) -> Result { + let app_name = self.app_name.trim(); + if app_name.is_empty() { + return Err(FoundryLocalError::InvalidConfiguration { + reason: "app_name must be set and non-empty".into(), + }); + } + + let cfg = NativeConfig::create(Arc::clone(api), app_name)?; + let c = api.config_api(); + + if let Some(dir) = &self.app_data_dir { + let s = to_cstring(dir)?; + // SAFETY: ptr is valid; the native call copies the string. + api.check(unsafe { (c.SetAppDataDir)(cfg.ptr, s.as_ptr()) })?; + } + if let Some(dir) = &self.model_cache_dir { + let s = to_cstring(dir)?; + api.check(unsafe { (c.SetModelCacheDir)(cfg.ptr, s.as_ptr()) })?; + } + if let Some(dir) = &self.logs_dir { + let s = to_cstring(dir)?; + api.check(unsafe { (c.SetLogsDir)(cfg.ptr, s.as_ptr()) })?; + } + if let Some(level) = self.log_level { + api.check(unsafe { (c.SetDefaultLogLevel)(cfg.ptr, level.as_native()) })?; + } + if let Some(urls) = &self.web_service_urls { + for url in urls.split(',').map(str::trim).filter(|u| !u.is_empty()) { + let s = to_cstring(url)?; + api.check(unsafe { (c.AddWebServiceEndpoint)(cfg.ptr, s.as_ptr()) })?; + } + } + if let Some(endpoint) = &self.service_endpoint { + let s = to_cstring(endpoint)?; + api.check(unsafe { (c.SetExternalServiceUrl)(cfg.ptr, s.as_ptr()) })?; + } + if let Some(extra) = &self.additional_settings { + if !extra.is_empty() { + let kvps = Kvps::from_pairs(Arc::clone(api), extra.iter())?; + api.check(unsafe { (c.SetAdditionalOptions)(cfg.ptr, kvps.as_ptr()) })?; + } + } + + Ok(cfg) + } +} + +/// Owning wrapper around a native `flConfiguration`, released on drop. +pub(crate) struct NativeConfig { + api: Arc, + ptr: *mut flConfiguration, +} + +impl NativeConfig { + fn create(api: Arc, app_name: &str) -> Result { + let name = to_cstring(app_name)?; + let mut ptr: *mut flConfiguration = std::ptr::null_mut(); + // SAFETY: `Create` writes a valid handle into `ptr` on success. + let status = unsafe { (api.config_api().Create)(name.as_ptr(), &mut ptr) }; + api.check(status)?; + Ok(Self { api, ptr }) + } + + pub(crate) fn as_ptr(&self) -> *const flConfiguration { + self.ptr + } +} + +impl Drop for NativeConfig { + fn drop(&mut self) { + if !self.ptr.is_null() { + // SAFETY: `ptr` was created by `Create` and not yet released. + unsafe { (self.api.config_api().Configuration_Release)(self.ptr) }; + self.ptr = std::ptr::null_mut(); + } + } +} diff --git a/sdk_v2/rust/src/detail/api.rs b/sdk_v2/rust/src/detail/api.rs new file mode 100644 index 000000000..5e6481254 --- /dev/null +++ b/sdk_v2/rust/src/detail/api.rs @@ -0,0 +1,431 @@ +//! Safe wrapper around the dynamically-loaded Foundry Local C ABI. +//! +//! [`Api`] loads the `foundry_local` shared library (pre-loading its ONNX Runtime +//! and GenAI dependencies first), resolves the root function table via +//! `FoundryLocalGetApi`, and caches the sub-API tables. It also provides the +//! `flStatus*` → [`FoundryLocalError`] mapping and small FFI string / key-value +//! helpers used throughout the `detail` layer. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use libloading::Library; + +use super::ffi::*; +use crate::error::{FoundryLocalError, Result}; + +// ── Library file names ─────────────────────────────────────────────────────── + +#[cfg(target_os = "windows")] +const LIB_FILE: &str = "foundry_local.dll"; +#[cfg(target_os = "macos")] +const LIB_FILE: &str = "libfoundry_local.dylib"; +#[cfg(all(unix, not(target_os = "macos")))] +const LIB_FILE: &str = "libfoundry_local.so"; + +#[cfg(target_os = "windows")] +const ORT_FILE: &str = "onnxruntime.dll"; +#[cfg(target_os = "macos")] +const ORT_FILE: &str = "libonnxruntime.dylib"; +#[cfg(all(unix, not(target_os = "macos")))] +const ORT_FILE: &str = "libonnxruntime.so"; + +#[cfg(target_os = "windows")] +const GENAI_FILE: &str = "onnxruntime-genai.dll"; +#[cfg(target_os = "macos")] +const GENAI_FILE: &str = "libonnxruntime-genai.dylib"; +#[cfg(all(unix, not(target_os = "macos")))] +const GENAI_FILE: &str = "libonnxruntime-genai.so"; + +// ── FFI string helpers ─────────────────────────────────────────────────────── + +/// Build a [`CString`], mapping an interior NUL to a validation error. +pub(crate) fn to_cstring(s: &str) -> Result { + CString::new(s).map_err(|e| FoundryLocalError::Validation { + reason: format!("string contains an interior NUL byte: {e}"), + }) +} + +/// Read a borrowed C string into an owned `String`. Returns `None` for null. +/// +/// # Safety +/// `ptr` must be null or point to a valid NUL-terminated string that stays alive +/// for the duration of this call. +pub(crate) unsafe fn cstr_to_string(ptr: *const c_char) -> Option { + if ptr.is_null() { + None + } else { + Some(CStr::from_ptr(ptr).to_string_lossy().into_owned()) + } +} + +fn map_error(code: flErrorCode, message: String) -> FoundryLocalError { + match code { + FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT | FOUNDRY_LOCAL_ERROR_INVALID_USAGE => { + FoundryLocalError::Validation { reason: message } + } + FOUNDRY_LOCAL_ERROR_INTERNAL | FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED => { + FoundryLocalError::Internal { reason: message } + } + FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED => FoundryLocalError::CommandExecution { + reason: if message.is_empty() { + "Operation cancelled".into() + } else { + message + }, + }, + _ => FoundryLocalError::CommandExecution { reason: message }, + } +} + +// ── Api ────────────────────────────────────────────────────────────────────── + +/// Loaded native library plus its resolved function tables. +/// +/// `Api` is `Send + Sync`: the underlying library is thread-safe for distinct +/// handles, and the cached vtable pointers are read-only for the process lifetime. +pub(crate) struct Api { + _foundry_lib: Library, + _dep_libs: Vec, + root: *const flApiVtable, + catalog: *const flCatalogApiVtable, + config: *const flConfigurationApiVtable, + item: *const flItemApiVtable, + inference: *const flInferenceApiVtable, + model: *const flModelApiVtable, +} + +// SAFETY: the native library is documented as thread-safe for independent +// handles; the cached vtable pointers are immutable for the library's lifetime. +unsafe impl Send for Api {} +unsafe impl Sync for Api {} + +impl std::fmt::Debug for Api { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Api").finish_non_exhaustive() + } +} + +impl Api { + /// Load the native library and resolve the API tables. + /// + /// `library_path` is an optional override pointing at the `foundry_local` + /// shared library file or the directory containing it. + pub(crate) fn load(library_path: Option<&str>) -> Result { + let (lib_path, native_dir) = resolve_library_path(library_path)?; + + // Pre-load ONNX Runtime then GenAI so the dynamic loader resolves + // foundry_local's dependencies regardless of rpath/search-path setup. + let dep_libs = preload_dependencies(native_dir.as_deref()); + + // SAFETY: `lib_path` was resolved from trusted configuration / build + // inputs. Loading a shared library executes foreign initialisers. + let foundry_lib = unsafe { + load_library(&lib_path).map_err(|e| match e { + FoundryLocalError::LibraryLoad { reason } => FoundryLocalError::LibraryLoad { + reason: format!( + "{reason}. Ensure the native Foundry Local library is available: set \ + FOUNDRY_LOCAL_LIB_DIR to the directory containing {LIB_FILE} (alongside its \ + onnxruntime and onnxruntime-genai libraries), pass it via \ + FoundryLocalConfig::library_path, or place it on the loader search path." + ), + }, + other => other, + })? + }; + + // SAFETY: the library exports `FoundryLocalGetApi` with this signature. + let root = unsafe { + let get_api: libloading::Symbol = foundry_lib + .get(FOUNDRY_LOCAL_GET_API_SYMBOL) + .map_err(|e| FoundryLocalError::LibraryLoad { + reason: format!("symbol 'FoundryLocalGetApi' not found: {e}"), + })?; + let ptr = get_api(FOUNDRY_LOCAL_API_VERSION); + if ptr.is_null() { + return Err(FoundryLocalError::LibraryLoad { + reason: format!( + "FoundryLocalGetApi({FOUNDRY_LOCAL_API_VERSION}) returned null (unsupported API version)" + ), + }); + } + ptr + }; + + // SAFETY: `root` is a valid, non-null vtable for the library's lifetime. + let root_ref = unsafe { &*root }; + let catalog = unsafe { (root_ref.GetCatalogApi)() }; + let config = unsafe { (root_ref.GetConfigurationApi)() }; + let item = unsafe { (root_ref.GetItemApi)() }; + let inference = unsafe { (root_ref.GetInferenceApi)() }; + let model = unsafe { (root_ref.GetModelApi)() }; + + for (name, ptr) in [ + ("CatalogApi", catalog as *const ()), + ("ConfigurationApi", config as *const ()), + ("ItemApi", item as *const ()), + ("InferenceApi", inference as *const ()), + ("ModelApi", model as *const ()), + ] { + if ptr.is_null() { + return Err(FoundryLocalError::LibraryLoad { + reason: format!("native {name} table is null"), + }); + } + } + + Ok(Self { + _foundry_lib: foundry_lib, + _dep_libs: dep_libs, + root, + catalog, + config, + item, + inference, + model, + }) + } + + #[inline] + pub(crate) fn root(&self) -> &flApiVtable { + // SAFETY: non-null and valid for the library's lifetime. + unsafe { &*self.root } + } + #[inline] + pub(crate) fn catalog_api(&self) -> &flCatalogApiVtable { + unsafe { &*self.catalog } + } + #[inline] + pub(crate) fn config_api(&self) -> &flConfigurationApiVtable { + unsafe { &*self.config } + } + #[inline] + pub(crate) fn item_api(&self) -> &flItemApiVtable { + unsafe { &*self.item } + } + #[inline] + pub(crate) fn inference_api(&self) -> &flInferenceApiVtable { + unsafe { &*self.inference } + } + #[inline] + pub(crate) fn model_api(&self) -> &flModelApiVtable { + unsafe { &*self.model } + } + + /// Convert a returned `flStatus*` into a `Result`. A non-null status is an error. + pub(crate) fn check(&self, status: flStatusPtr) -> Result<()> { + if status.is_null() { + return Ok(()); + } + let root = self.root(); + // SAFETY: `status` is a valid non-null status owned by us until released. + unsafe { + let code = (root.Status_GetErrorCode)(status); + let message = cstr_to_string((root.Status_GetErrorMessage)(status)).unwrap_or_default(); + (root.Status_Release)(status); + Err(map_error(code, message)) + } + } + + /// Read a returned `flStatus*` as an optional message. `None` for success + /// (null status). Releases the status. Used where a non-null status is a + /// soft/partial failure rather than a hard error (e.g. EP registration). + pub(crate) fn status_message(&self, status: flStatusPtr) -> Option { + if status.is_null() { + return None; + } + let root = self.root(); + // SAFETY: `status` is a valid non-null status owned by us until released. + unsafe { + let message = cstr_to_string((root.Status_GetErrorMessage)(status)).unwrap_or_default(); + (root.Status_Release)(status); + Some(message) + } + } +} + +// ── KeyValuePairs RAII helper ──────────────────────────────────────────────── + +/// Owning wrapper around a native `flKeyValuePairs`, released on drop. +pub(crate) struct Kvps { + api: Arc, + ptr: *mut flKeyValuePairs, +} + +impl Kvps { + /// Create an empty key/value collection. + pub(crate) fn new(api: Arc) -> Self { + let mut ptr: *mut flKeyValuePairs = std::ptr::null_mut(); + // SAFETY: `CreateKeyValuePairs` writes a valid handle into `ptr`. + unsafe { (api.root().CreateKeyValuePairs)(&mut ptr) }; + Self { api, ptr } + } + + /// Build from an iterator of `(key, value)` string pairs. + pub(crate) fn from_pairs(api: Arc, pairs: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + let mut kvps = Self::new(api); + for (k, v) in pairs { + kvps.set(k.as_ref(), v.as_ref())?; + } + Ok(kvps) + } + + /// Add or replace a key/value pair. + pub(crate) fn set(&mut self, key: &str, value: &str) -> Result<()> { + let key = to_cstring(key)?; + let value = to_cstring(value)?; + // SAFETY: the native call copies both strings; our CStrings outlive the call. + unsafe { (self.api.root().AddKeyValuePair)(self.ptr, key.as_ptr(), value.as_ptr()) }; + Ok(()) + } + + pub(crate) fn as_ptr(&self) -> *const flKeyValuePairs { + self.ptr + } +} + +impl Drop for Kvps { + fn drop(&mut self) { + if !self.ptr.is_null() { + // SAFETY: `ptr` was created by `CreateKeyValuePairs` and not yet released. + unsafe { (self.api.root().KeyValuePairs_Release)(self.ptr) }; + self.ptr = std::ptr::null_mut(); + } + } +} + +/// Read a borrowed native `flKeyValuePairs` into an owned `Vec<(String, Option)>`. +/// +/// # Safety +/// `kvps` must be null or a valid pointer that stays alive for the duration of this call. +pub(crate) unsafe fn read_kvps( + api: &Api, + kvps: *const flKeyValuePairs, +) -> Vec<(String, Option)> { + if kvps.is_null() { + return Vec::new(); + } + let mut keys: *const *const c_char = std::ptr::null(); + let mut values: *const *const c_char = std::ptr::null(); + let mut count: usize = 0; + (api.root().GetKeyValuePairs)(kvps, &mut keys, &mut values, &mut count); + if keys.is_null() || count == 0 { + return Vec::new(); + } + let mut out = Vec::with_capacity(count); + for i in 0..count { + let key = cstr_to_string(*keys.add(i)).unwrap_or_default(); + let value = if values.is_null() { + None + } else { + cstr_to_string(*values.add(i)) + }; + out.push((key, value)); + } + out +} + +// ── Library discovery & loading ────────────────────────────────────────────── + +#[cfg(unix)] +unsafe fn load_library(path: &Path) -> Result { + use libloading::os::unix::{Library as UnixLibrary, RTLD_GLOBAL, RTLD_NOW}; + UnixLibrary::open(Some(path), RTLD_NOW | RTLD_GLOBAL) + .map(Library::from) + .map_err(|e| FoundryLocalError::LibraryLoad { + reason: format!("failed to load native library at {}: {e}", path.display()), + }) +} + +#[cfg(windows)] +unsafe fn load_library(path: &Path) -> Result { + Library::new(path).map_err(|e| FoundryLocalError::LibraryLoad { + reason: format!("failed to load native library at {}: {e}", path.display()), + }) +} + +/// Best-effort pre-load of ORT and GenAI from the native directory. Failures are +/// ignored: foundry_local may resolve them via rpath/search path instead. +fn preload_dependencies(native_dir: Option<&Path>) -> Vec { + let mut libs = Vec::new(); + let Some(dir) = native_dir else { + return libs; + }; + + #[allow(unused_mut)] + let mut deps: Vec<&str> = vec![ORT_FILE, GENAI_FILE]; + #[cfg(all(windows, feature = "winml"))] + deps.push("Microsoft.Windows.AI.MachineLearning.dll"); + + // Help GenAI's dlopen build find the exact ORT we are pre-loading. + let ort_path = dir.join(ORT_FILE); + if ort_path.exists() && std::env::var_os("ORT_LIB_PATH").is_none() { + std::env::set_var("ORT_LIB_PATH", &ort_path); + } + + for dep in deps { + let dep_path = dir.join(dep); + if dep_path.exists() { + // SAFETY: pre-loading a known dependency from the trusted native dir. + if let Ok(lib) = unsafe { load_library(&dep_path) } { + libs.push(lib); + } + } + } + libs +} + +/// Resolve the full path to the `foundry_local` library and its containing dir. +/// +/// Search order: +/// 1. `library_path` override (a file or a directory). +/// 2. `FOUNDRY_LOCAL_LIB_DIR` environment variable (dev override). +/// 3. `FOUNDRY_NATIVE_DIR` compile-time path (set by `build.rs`). +/// 4. The directory of the current executable. +/// 5. Bare library name resolved via the system search path. +fn resolve_library_path(library_path: Option<&str>) -> Result<(PathBuf, Option)> { + // 1. Explicit override: a file path or a directory. + if let Some(p) = library_path { + let path = Path::new(p); + if path.is_file() { + let dir = path.parent().map(Path::to_path_buf); + return Ok((path.to_path_buf(), dir)); + } + let candidate = path.join(LIB_FILE); + if candidate.is_file() { + return Ok((candidate, Some(path.to_path_buf()))); + } + } + + let mut dirs: Vec = Vec::new(); + if let Ok(dir) = std::env::var("FOUNDRY_LOCAL_LIB_DIR") { + if !dir.trim().is_empty() { + dirs.push(PathBuf::from(dir)); + } + } + if let Some(dir) = option_env!("FOUNDRY_NATIVE_DIR") { + dirs.push(PathBuf::from(dir)); + } + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + dirs.push(dir.to_path_buf()); + } + } + + for dir in &dirs { + let candidate = dir.join(LIB_FILE); + if candidate.is_file() { + return Ok((candidate, Some(dir.clone()))); + } + } + + // 5. Fall back to the bare library name (system loader search path). + Ok((PathBuf::from(LIB_FILE), None)) +} diff --git a/sdk_v2/rust/src/detail/ffi.rs b/sdk_v2/rust/src/detail/ffi.rs new file mode 100644 index 000000000..22645fa88 --- /dev/null +++ b/sdk_v2/rust/src/detail/ffi.rs @@ -0,0 +1,743 @@ +//! Raw FFI declarations mirroring `sdk_v2/cpp/include/foundry_local/foundry_local_c.h`. +//! +//! This module is a faithful `repr(C)` transcription of the Foundry Local C ABI. +//! It exposes the opaque handle types, versioned data structs, enums, callback +//! signatures, and the six function-pointer tables that the library returns via +//! [`FoundryLocalGetApi`]. +//! +//! Everything here is `unsafe` to use; safe wrappers live in [`super::api`] and the +//! higher-level detail modules. +#![allow(non_snake_case)] +#![allow(non_camel_case_types)] +#![allow(dead_code)] + +use core::ffi::c_void; +use std::os::raw::{c_char, c_int}; + +/// The library is built against this API version (`FOUNDRY_LOCAL_API_VERSION`). +pub const FOUNDRY_LOCAL_API_VERSION: u32 = 1; + +// ── Opaque handle types ────────────────────────────────────────────────────── + +macro_rules! opaque_type { + ($name:ident) => { + #[repr(C)] + pub struct $name { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, + } + }; +} + +opaque_type!(flCatalog); +opaque_type!(flConfiguration); +opaque_type!(flItem); +opaque_type!(flItemQueue); +opaque_type!(flKeyValuePairs); +opaque_type!(flManager); +opaque_type!(flModel); +opaque_type!(flModelInfo); +opaque_type!(flModelList); +opaque_type!(flRequest); +opaque_type!(flResponse); +opaque_type!(flSession); +opaque_type!(flStatus); + +/// A non-null `flStatus*` indicates an error; `null` indicates success. +pub type flStatusPtr = *mut flStatus; + +// ── Enums (C enums are `int`-sized) ────────────────────────────────────────── + +pub type flErrorCode = c_int; +pub const FOUNDRY_LOCAL_OK: flErrorCode = 0; +pub const FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED: flErrorCode = 1; +pub const FOUNDRY_LOCAL_ERROR_INTERNAL: flErrorCode = 2; +pub const FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT: flErrorCode = 3; +pub const FOUNDRY_LOCAL_ERROR_INVALID_USAGE: flErrorCode = 4; +pub const FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED: flErrorCode = 5; +pub const FOUNDRY_LOCAL_ERROR_NETWORK: flErrorCode = 6; + +pub type flLogLevel = c_int; +pub const FOUNDRY_LOCAL_LOG_VERBOSE: flLogLevel = 0; +pub const FOUNDRY_LOCAL_LOG_DEBUG: flLogLevel = 1; +pub const FOUNDRY_LOCAL_LOG_INFO: flLogLevel = 2; +pub const FOUNDRY_LOCAL_LOG_WARNING: flLogLevel = 3; +pub const FOUNDRY_LOCAL_LOG_ERROR: flLogLevel = 4; +pub const FOUNDRY_LOCAL_LOG_FATAL: flLogLevel = 5; + +pub type flDeviceType = c_int; +pub const FOUNDRY_LOCAL_DEVICE_NOTSET: flDeviceType = 0; +pub const FOUNDRY_LOCAL_DEVICE_CPU: flDeviceType = 1; +pub const FOUNDRY_LOCAL_DEVICE_GPU: flDeviceType = 2; +pub const FOUNDRY_LOCAL_DEVICE_NPU: flDeviceType = 3; + +pub type flTensorDataType = c_int; +pub const FOUNDRY_LOCAL_TENSOR_UNDEFINED: flTensorDataType = 0; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT: flTensorDataType = 1; +pub const FOUNDRY_LOCAL_TENSOR_UINT8: flTensorDataType = 2; +pub const FOUNDRY_LOCAL_TENSOR_INT8: flTensorDataType = 3; +pub const FOUNDRY_LOCAL_TENSOR_UINT16: flTensorDataType = 4; +pub const FOUNDRY_LOCAL_TENSOR_INT16: flTensorDataType = 5; +pub const FOUNDRY_LOCAL_TENSOR_INT32: flTensorDataType = 6; +pub const FOUNDRY_LOCAL_TENSOR_INT64: flTensorDataType = 7; +pub const FOUNDRY_LOCAL_TENSOR_STRING: flTensorDataType = 8; +pub const FOUNDRY_LOCAL_TENSOR_BOOL: flTensorDataType = 9; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT16: flTensorDataType = 10; +pub const FOUNDRY_LOCAL_TENSOR_DOUBLE: flTensorDataType = 11; +pub const FOUNDRY_LOCAL_TENSOR_UINT32: flTensorDataType = 12; +pub const FOUNDRY_LOCAL_TENSOR_UINT64: flTensorDataType = 13; +pub const FOUNDRY_LOCAL_TENSOR_COMPLEX64: flTensorDataType = 14; +pub const FOUNDRY_LOCAL_TENSOR_COMPLEX128: flTensorDataType = 15; +pub const FOUNDRY_LOCAL_TENSOR_BFLOAT16: flTensorDataType = 16; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FN: flTensorDataType = 17; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FNUZ: flTensorDataType = 18; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2: flTensorDataType = 19; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2FNUZ: flTensorDataType = 20; +pub const FOUNDRY_LOCAL_TENSOR_UINT4: flTensorDataType = 21; +pub const FOUNDRY_LOCAL_TENSOR_INT4: flTensorDataType = 22; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT4E2M1: flTensorDataType = 23; +pub const FOUNDRY_LOCAL_TENSOR_FLOAT8E8M0: flTensorDataType = 24; + +pub type flItemType = c_int; +pub const FOUNDRY_LOCAL_ITEM_UNKNOWN: flItemType = 0; +pub const FOUNDRY_LOCAL_ITEM_BYTES: flItemType = 1; +pub const FOUNDRY_LOCAL_ITEM_TENSOR: flItemType = 10; +pub const FOUNDRY_LOCAL_ITEM_TEXT: flItemType = 20; +pub const FOUNDRY_LOCAL_ITEM_MESSAGE: flItemType = 21; +pub const FOUNDRY_LOCAL_ITEM_IMAGE: flItemType = 25; +pub const FOUNDRY_LOCAL_ITEM_AUDIO: flItemType = 30; +pub const FOUNDRY_LOCAL_ITEM_SPEECH_SEGMENT: flItemType = 31; +pub const FOUNDRY_LOCAL_ITEM_SPEECH_RESULT: flItemType = 32; +pub const FOUNDRY_LOCAL_ITEM_TOOL_CALL: flItemType = 100; +pub const FOUNDRY_LOCAL_ITEM_TOOL_RESULT: flItemType = 101; +pub const FOUNDRY_LOCAL_ITEM_QUEUE: flItemType = 200; + +pub type flTextItemType = c_int; +pub const FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT: flTextItemType = 0; +pub const FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING: flTextItemType = 1; +pub const FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON: flTextItemType = 2; + +pub type flMessageRole = c_int; +pub const FOUNDRY_LOCAL_ROLE_NONE: flMessageRole = 0; +pub const FOUNDRY_LOCAL_ROLE_SYSTEM: flMessageRole = 1; +pub const FOUNDRY_LOCAL_ROLE_USER: flMessageRole = 2; +pub const FOUNDRY_LOCAL_ROLE_ASSISTANT: flMessageRole = 3; +pub const FOUNDRY_LOCAL_ROLE_TOOL: flMessageRole = 4; +pub const FOUNDRY_LOCAL_ROLE_DEVELOPER: flMessageRole = 5; + +pub type flFinishReason = c_int; +pub const FOUNDRY_LOCAL_FINISH_NONE: flFinishReason = 0; +pub const FOUNDRY_LOCAL_FINISH_ERROR: flFinishReason = 1; +pub const FOUNDRY_LOCAL_FINISH_STOP: flFinishReason = 2; +pub const FOUNDRY_LOCAL_FINISH_LENGTH: flFinishReason = 3; +pub const FOUNDRY_LOCAL_FINISH_TOOL_CALLS: flFinishReason = 4; + +pub type flToolChoice = c_int; +pub const FOUNDRY_LOCAL_TOOL_CHOICE_AUTO: flToolChoice = 0; +pub const FOUNDRY_LOCAL_TOOL_CHOICE_NONE: flToolChoice = 1; +pub const FOUNDRY_LOCAL_TOOL_CHOICE_REQUIRED: flToolChoice = 2; + +// ── Well-known property / parameter keys ───────────────────────────────────── + +pub const FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR: &str = "display_name"; +pub const FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR: &str = "type"; +pub const FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR: &str = "publisher"; +pub const FOUNDRY_LOCAL_MODEL_PROP_LICENSE_STR: &str = "license"; +pub const FOUNDRY_LOCAL_MODEL_PROP_LICENSE_DESCRIPTION_STR: &str = "license_description"; +pub const FOUNDRY_LOCAL_MODEL_PROP_TASK_STR: &str = "task"; +pub const FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR: &str = "model_provider"; +pub const FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR: &str = "min_fl_version"; +pub const FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR: &str = "input_modalities"; +pub const FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR: &str = "output_modalities"; +pub const FOUNDRY_LOCAL_MODEL_PROP_CAPABILITIES_STR: &str = "capabilities"; + +pub const FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT: &str = "supports_tool_calling"; +pub const FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT: &str = "supports_reasoning"; +pub const FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT: &str = "filesize_mb"; +pub const FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT: &str = "max_output_tokens"; +pub const FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT: &str = "created_at_unix"; +pub const FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT: &str = "context_length"; + +pub const FOUNDRY_LOCAL_PARAM_TEMPERATURE: &str = "temperature"; +pub const FOUNDRY_LOCAL_PARAM_TOP_P: &str = "top_p"; +pub const FOUNDRY_LOCAL_PARAM_TOP_K: &str = "top_k"; +pub const FOUNDRY_LOCAL_PARAM_MAX_OUTPUT_TOKENS: &str = "max_output_tokens"; +pub const FOUNDRY_LOCAL_PARAM_FREQUENCY_PENALTY: &str = "frequency_penalty"; +pub const FOUNDRY_LOCAL_PARAM_PRESENCE_PENALTY: &str = "presence_penalty"; +pub const FOUNDRY_LOCAL_PARAM_SEED: &str = "seed"; +pub const FOUNDRY_LOCAL_PARAM_EARLY_STOPPING: &str = "early_stopping"; +pub const FOUNDRY_LOCAL_PARAM_DO_SAMPLE: &str = "do_sample"; +pub const FOUNDRY_LOCAL_PARAM_TOOL_CHOICE: &str = "tool_choice"; + +// ── Versioned data structs ─────────────────────────────────────────────────── + +#[repr(C)] +pub struct flUsage { + pub version: u32, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub total_tokens: i64, +} + +#[repr(C)] +pub struct flEpInfo { + pub version: u32, + pub name: *const c_char, + pub is_registered: bool, +} + +pub type flBytesDataDeleter = + Option; +pub type flTensorDataDeleter = + Option; +pub type flImageDataDeleter = + Option; +pub type flAudioDataDeleter = + Option; + +#[repr(C)] +pub struct flTextData { + pub version: u32, + pub text: *const c_char, + pub r#type: flTextItemType, +} + +#[repr(C)] +pub struct flBytesData { + pub version: u32, + pub item_type: flItemType, + pub data: *const c_void, + pub mutable_data: *mut c_void, + pub data_size: usize, + pub deleter: flBytesDataDeleter, + pub deleter_user_data: *mut c_void, +} + +#[repr(C)] +pub struct flTensorData { + pub version: u32, + pub data_type: flTensorDataType, + pub data: *const c_void, + pub mutable_data: *mut c_void, + pub shape: *const i64, + pub rank: usize, + pub deleter: flTensorDataDeleter, + pub deleter_user_data: *mut c_void, +} + +#[repr(C)] +pub struct flMessageData { + pub version: u32, + pub role: flMessageRole, + pub content_items: *const *const flItem, + pub content_items_count: usize, + pub name: *const c_char, +} + +#[repr(C)] +pub struct flImageData { + pub version: u32, + pub data: *const c_void, + pub mutable_data: *mut c_void, + pub data_size: usize, + pub format: *const c_char, + pub uri: *const c_char, + pub deleter: flImageDataDeleter, + pub deleter_user_data: *mut c_void, +} + +#[repr(C)] +pub struct flAudioData { + pub version: u32, + pub data: *const c_void, + pub mutable_data: *mut c_void, + pub data_size: usize, + pub format: *const c_char, + pub uri: *const c_char, + pub sample_rate: c_int, + pub channels: c_int, + pub deleter: flAudioDataDeleter, + pub deleter_user_data: *mut c_void, +} + +#[repr(C)] +pub struct flToolCallData { + pub version: u32, + pub call_id: *const c_char, + pub name: *const c_char, + pub arguments: *const c_char, +} + +#[repr(C)] +pub struct flToolResultData { + pub version: u32, + pub call_id: *const c_char, + pub result: *const c_char, +} + +// ── Speech recognition output types (output-only; see foundry_local_c.h) ────── + +/// Sentinel for absent time fields in the speech structs (`INT64_MIN`). +pub const FOUNDRY_LOCAL_DURATION_UNSET: i64 = i64::MIN; +/// Sentinel for absent confidence in `flSpeechWord` (`-FLT_MAX`). +pub const FOUNDRY_LOCAL_CONFIDENCE_UNSET: f32 = f32::MIN; + +pub type flSpeechSegmentKind = c_int; +pub const FOUNDRY_LOCAL_SPEECH_SEGMENT_NONE: flSpeechSegmentKind = 0; +pub const FOUNDRY_LOCAL_SPEECH_SEGMENT_PARTIAL: flSpeechSegmentKind = 1; +pub const FOUNDRY_LOCAL_SPEECH_SEGMENT_FINAL: flSpeechSegmentKind = 2; + +#[repr(C)] +pub struct flSpeechWord { + pub version: u32, + pub text: *const c_char, + pub start_time_ms: i64, + pub end_time_ms: i64, + pub confidence: f32, + pub speaker_id: *const c_char, +} + +#[repr(C)] +pub struct flSpeechSegmentData { + pub version: u32, + pub kind: flSpeechSegmentKind, + pub text: *const c_char, + pub start_time_ms: i64, + pub end_time_ms: i64, + pub utterance_start: bool, + pub words: *const flSpeechWord, + pub words_count: usize, + pub language: *const c_char, +} + +#[repr(C)] +pub struct flSpeechResultData { + pub version: u32, + pub text: *const c_char, + pub language: *const c_char, + pub duration_ms: i64, + pub segments: *const *const flItem, + pub segments_count: usize, +} + +#[repr(C)] +pub struct flStreamingCallbackData { + pub version: u32, + pub item_queue: *mut flItemQueue, +} + +#[repr(C)] +pub struct flToolDefinition { + pub version: u32, + pub name: *const c_char, + pub description: *const c_char, + pub json_schema: *const c_char, +} + +// ── Callback types (plain C calling convention) ────────────────────────────── + +pub type flProgressCallback = + Option c_int>; +pub type flStreamingCallback = + Option c_int>; +pub type flEpProgressCallback = Option< + unsafe extern "C" fn(ep_name: *const c_char, value: f32, user_data: *mut c_void) -> c_int, +>; + +// ── Exported entry points (FL_API_CALL == __stdcall on Win32) ──────────────── +// +// The library is loaded at runtime via `libloading`; these are the signatures of +// the two exported symbols resolved from it. `flApi` is the root vtable type. + +pub type flApi = flApiVtable; + +pub type FoundryLocalGetApiFn = unsafe extern "system" fn(version: u32) -> *const flApiVtable; +pub type FoundryLocalGetVersionStringFn = unsafe extern "system" fn() -> *const c_char; + +pub const FOUNDRY_LOCAL_GET_API_SYMBOL: &[u8] = b"FoundryLocalGetApi\0"; +pub const FOUNDRY_LOCAL_GET_VERSION_STRING_SYMBOL: &[u8] = b"FoundryLocalGetVersionString\0"; + +// ── Function tables ────────────────────────────────────────────────────────── +// +// Field order and signatures MUST match foundry_local_c.h exactly — the tables +// are consumed positionally, so a new entry inserted mid-table upstream must be +// mirrored at the same position here (e.g. GetSpeechSegment/GetSpeechResult sit +// between GetToolResult and GetMetadata in flItemApi). + +/// Root API table (`flApi`). +#[repr(C)] +pub struct flApiVtable { + // Status + pub Status_Create: + unsafe extern "system" fn(error_code: flErrorCode, error_msg: *const c_char) -> flStatusPtr, + pub Status_Release: unsafe extern "system" fn(instance: *mut flStatus), + pub Status_GetErrorCode: unsafe extern "system" fn(status: *const flStatus) -> flErrorCode, + pub Status_GetErrorMessage: unsafe extern "system" fn(status: *const flStatus) -> *const c_char, + + // Manager lifecycle + pub Manager_Create: unsafe extern "system" fn( + config: *const flConfiguration, + out_manager: *mut *mut flManager, + ) -> flStatusPtr, + pub Manager_Release: unsafe extern "system" fn(instance: *mut flManager), + + pub Manager_GetCatalog: unsafe extern "system" fn( + manager: *const flManager, + out_catalog: *mut *mut flCatalog, + ) -> flStatusPtr, + pub Manager_WebServiceStart: unsafe extern "system" fn(manager: *mut flManager) -> flStatusPtr, + pub Manager_WebServiceUrls: unsafe extern "system" fn( + manager: *const flManager, + out_urls: *mut *const *const c_char, + out_num_urls: *mut usize, + ) -> flStatusPtr, + pub Manager_WebServiceStop: unsafe extern "system" fn(manager: *mut flManager) -> flStatusPtr, + + // Sub-API accessors + pub GetCatalogApi: unsafe extern "system" fn() -> *const flCatalogApiVtable, + pub GetConfigurationApi: unsafe extern "system" fn() -> *const flConfigurationApiVtable, + pub GetItemApi: unsafe extern "system" fn() -> *const flItemApiVtable, + pub GetInferenceApi: unsafe extern "system" fn() -> *const flInferenceApiVtable, + pub GetModelApi: unsafe extern "system" fn() -> *const flModelApiVtable, + + // KeyValuePairs + pub CreateKeyValuePairs: unsafe extern "system" fn(out: *mut *mut flKeyValuePairs), + pub AddKeyValuePair: unsafe extern "system" fn( + kvps: *mut flKeyValuePairs, + key: *const c_char, + value: *const c_char, + ), + pub GetKeyValue: unsafe extern "system" fn( + kvps: *const flKeyValuePairs, + key: *const c_char, + ) -> *const c_char, + pub GetKeyValuePairs: unsafe extern "system" fn( + kvps: *const flKeyValuePairs, + keys: *mut *const *const c_char, + values: *mut *const *const c_char, + num_entries: *mut usize, + ), + pub RemoveKeyValuePair: + unsafe extern "system" fn(kvps: *mut flKeyValuePairs, key: *const c_char), + pub KeyValuePairs_Release: unsafe extern "system" fn(instance: *mut flKeyValuePairs), + + // ModelList + pub ModelList_Release: unsafe extern "system" fn(instance: *mut flModelList), + pub ModelList_Size: unsafe extern "system" fn(models: *const flModelList) -> usize, + pub ModelList_GetAt: + unsafe extern "system" fn(models: *const flModelList, idx: usize) -> *mut flModel, + + // EP detection + pub Manager_GetDiscoverableEps: unsafe extern "system" fn( + manager: *const flManager, + out_eps: *mut *const flEpInfo, + out_count: *mut usize, + ) -> flStatusPtr, + pub Manager_DownloadAndRegisterEps: unsafe extern "system" fn( + manager: *mut flManager, + ep_names: *const *const c_char, + num_ep_names: usize, + callback: flEpProgressCallback, + user_data: *mut c_void, + ) -> flStatusPtr, + pub Manager_IsEpDownloadInProgress: + unsafe extern "system" fn(manager: *const flManager) -> bool, + + pub Manager_Shutdown: unsafe extern "system" fn(manager: *mut flManager) -> flStatusPtr, + pub Manager_IsShutdownRequested: unsafe extern "system" fn(manager: *const flManager) -> bool, +} + +/// Item API table (`flItemApi`). +#[repr(C)] +pub struct flItemApiVtable { + pub Create: + unsafe extern "system" fn(item_type: flItemType, out_item: *mut *mut flItem) -> flStatusPtr, + pub Item_Release: unsafe extern "system" fn(instance: *mut flItem), + pub GetType: unsafe extern "system" fn(item: *const flItem) -> flItemType, + + pub SetBytes: + unsafe extern "system" fn(item: *mut flItem, bytes: *const flBytesData) -> flStatusPtr, + pub SetTensor: + unsafe extern "system" fn(item: *mut flItem, tensor: *const flTensorData) -> flStatusPtr, + pub SetText: + unsafe extern "system" fn(item: *mut flItem, text_data: *const flTextData) -> flStatusPtr, + pub SetMessage: + unsafe extern "system" fn(item: *mut flItem, message: *const flMessageData) -> flStatusPtr, + pub SetImage: + unsafe extern "system" fn(item: *mut flItem, image: *const flImageData) -> flStatusPtr, + pub SetAudio: + unsafe extern "system" fn(item: *mut flItem, audio: *const flAudioData) -> flStatusPtr, + pub SetToolCall: unsafe extern "system" fn( + item: *mut flItem, + tool_call: *const flToolCallData, + ) -> flStatusPtr, + pub SetToolResult: unsafe extern "system" fn( + item: *mut flItem, + tool_result: *const flToolResultData, + ) -> flStatusPtr, + + pub GetBytes: + unsafe extern "system" fn(item: *const flItem, out_bytes: *mut flBytesData) -> flStatusPtr, + pub GetText: unsafe extern "system" fn( + item: *const flItem, + out_text_data: *mut flTextData, + ) -> flStatusPtr, + pub GetMessage: unsafe extern "system" fn( + item: *const flItem, + out_message: *mut flMessageData, + ) -> flStatusPtr, + pub GetTensor: unsafe extern "system" fn( + item: *const flItem, + out_tensor: *mut flTensorData, + ) -> flStatusPtr, + pub GetImage: + unsafe extern "system" fn(item: *const flItem, out_image: *mut flImageData) -> flStatusPtr, + pub GetAudio: + unsafe extern "system" fn(item: *const flItem, out_audio: *mut flAudioData) -> flStatusPtr, + pub GetToolCall: unsafe extern "system" fn( + item: *const flItem, + out_tool_call: *mut flToolCallData, + ) -> flStatusPtr, + pub GetToolResult: unsafe extern "system" fn( + item: *const flItem, + out_tool_result: *mut flToolResultData, + ) -> flStatusPtr, + + pub GetSpeechSegment: unsafe extern "system" fn( + item: *const flItem, + out_segment: *mut flSpeechSegmentData, + ) -> flStatusPtr, + pub GetSpeechResult: unsafe extern "system" fn( + item: *const flItem, + out_result: *mut flSpeechResultData, + ) -> flStatusPtr, + + pub GetMetadata: unsafe extern "system" fn( + item: *const flItem, + out_metadata: *mut *const flKeyValuePairs, + ) -> flStatusPtr, + pub GetMutableMetadata: unsafe extern "system" fn( + item: *mut flItem, + out_metadata: *mut *mut flKeyValuePairs, + ) -> flStatusPtr, + pub GetQueue: unsafe extern "system" fn( + item: *mut flItem, + out_queue: *mut *mut flItemQueue, + ) -> flStatusPtr, + + // ItemQueue + pub ItemQueue_Create: + unsafe extern "system" fn(out_queue: *mut *mut flItemQueue) -> flStatusPtr, + pub ItemQueue_Release: unsafe extern "system" fn(instance: *mut flItemQueue), + pub ItemQueue_Push: + unsafe extern "system" fn(queue: *mut flItemQueue, item: *mut flItem) -> flStatusPtr, + pub ItemQueue_TryPop: + unsafe extern "system" fn(queue: *mut flItemQueue, out_item: *mut *mut flItem) -> bool, + pub ItemQueue_Size: unsafe extern "system" fn(queue: *const flItemQueue) -> usize, + pub ItemQueue_MarkFinished: unsafe extern "system" fn(queue: *mut flItemQueue), + pub ItemQueue_IsFinished: unsafe extern "system" fn(queue: *const flItemQueue) -> bool, +} + +/// Inference API table (`flInferenceApi`). +#[repr(C)] +pub struct flInferenceApiVtable { + pub Request_Create: unsafe extern "system" fn(out_request: *mut *mut flRequest) -> flStatusPtr, + pub Request_Release: unsafe extern "system" fn(instance: *mut flRequest), + pub Request_AddItem: unsafe extern "system" fn( + request: *mut flRequest, + item: *mut flItem, + take_ownership: bool, + ) -> flStatusPtr, + pub Request_GetItemCount: unsafe extern "system" fn(request: *const flRequest) -> usize, + pub Request_GetItem: unsafe extern "system" fn( + request: *const flRequest, + idx: usize, + out_item: *mut *const flItem, + ) -> flStatusPtr, + pub Request_SetOptions: unsafe extern "system" fn( + request: *mut flRequest, + options: *const flKeyValuePairs, + ) -> flStatusPtr, + pub Request_Cancel: unsafe extern "system" fn(request: *mut flRequest) -> flStatusPtr, + + pub Response_Create: + unsafe extern "system" fn(out_response: *mut *mut flResponse) -> flStatusPtr, + pub Response_Release: unsafe extern "system" fn(instance: *mut flResponse), + pub Response_GetItemCount: unsafe extern "system" fn(response: *const flResponse) -> usize, + pub Response_GetItem: unsafe extern "system" fn( + response: *const flResponse, + idx: usize, + out_item: *mut *const flItem, + ) -> flStatusPtr, + pub Response_GetFinishReason: + unsafe extern "system" fn(response: *const flResponse) -> flFinishReason, + pub Response_GetUsage: unsafe extern "system" fn( + response: *const flResponse, + out_usage: *mut flUsage, + ) -> flStatusPtr, + + pub Session_Create: unsafe extern "system" fn( + model: *const flModel, + out_session: *mut *mut flSession, + ) -> flStatusPtr, + pub Session_Release: unsafe extern "system" fn(instance: *mut flSession), + pub Session_SetStreamingCallback: unsafe extern "system" fn( + session: *mut flSession, + callback: flStreamingCallback, + user_data: *mut c_void, + ) -> flStatusPtr, + pub Session_SetOptions: unsafe extern "system" fn( + session: *mut flSession, + options: *const flKeyValuePairs, + ) -> flStatusPtr, + pub Session_ProcessRequest: unsafe extern "system" fn( + session: *mut flSession, + request: *const flRequest, + response: *mut *mut flResponse, + ) -> flStatusPtr, + pub Session_AddToolDefinition: unsafe extern "system" fn( + session: *mut flSession, + tool_def: *const flToolDefinition, + ) -> flStatusPtr, + pub Session_RemoveToolDefinition: unsafe extern "system" fn( + session: *mut flSession, + tool_name: *const c_char, + out_removed: *mut bool, + ) -> flStatusPtr, + pub Session_GetTurnCount: unsafe extern "system" fn(session: *const flSession) -> usize, + pub Session_UndoTurns: + unsafe extern "system" fn(session: *mut flSession, count: usize) -> flStatusPtr, +} + +/// Configuration API table (`flConfigurationApi`). +#[repr(C)] +pub struct flConfigurationApiVtable { + pub Create: unsafe extern "system" fn( + app_name: *const c_char, + out_config: *mut *mut flConfiguration, + ) -> flStatusPtr, + pub Configuration_Release: unsafe extern "system" fn(instance: *mut flConfiguration), + pub SetDefaultLogLevel: + unsafe extern "system" fn(config: *mut flConfiguration, level: flLogLevel) -> flStatusPtr, + pub SetAppDataDir: + unsafe extern "system" fn(config: *mut flConfiguration, dir: *const c_char) -> flStatusPtr, + pub SetLogsDir: + unsafe extern "system" fn(config: *mut flConfiguration, dir: *const c_char) -> flStatusPtr, + pub SetModelCacheDir: + unsafe extern "system" fn(config: *mut flConfiguration, dir: *const c_char) -> flStatusPtr, + pub AddCatalogUrl: unsafe extern "system" fn( + config: *mut flConfiguration, + url: *const c_char, + filter_override: *const c_char, + ) -> flStatusPtr, + pub SetCatalogRegion: unsafe extern "system" fn( + config: *mut flConfiguration, + region: *const c_char, + ) -> flStatusPtr, + pub AddWebServiceEndpoint: + unsafe extern "system" fn(config: *mut flConfiguration, url: *const c_char) -> flStatusPtr, + pub SetExternalServiceUrl: + unsafe extern "system" fn(config: *mut flConfiguration, url: *const c_char) -> flStatusPtr, + pub SetAdditionalOptions: unsafe extern "system" fn( + config: *mut flConfiguration, + options: *const flKeyValuePairs, + ) -> flStatusPtr, +} + +/// Catalog API table (`flCatalogApi`). +#[repr(C)] +pub struct flCatalogApiVtable { + pub GetName: unsafe extern "system" fn( + catalog: *const flCatalog, + out_name: *mut *const c_char, + ) -> flStatusPtr, + pub GetModels: unsafe extern "system" fn( + catalog: *const flCatalog, + out_models: *mut *mut flModelList, + ) -> flStatusPtr, + pub GetModel: unsafe extern "system" fn( + catalog: *const flCatalog, + alias: *const c_char, + out_model: *mut *mut flModel, + ) -> flStatusPtr, + pub GetModelVariant: unsafe extern "system" fn( + catalog: *const flCatalog, + model_id: *const c_char, + out_model: *mut *mut flModel, + ) -> flStatusPtr, + pub GetLatestVersion: unsafe extern "system" fn( + catalog: *const flCatalog, + model: *const flModel, + out_model: *mut *mut flModel, + ) -> flStatusPtr, + pub GetCachedModels: unsafe extern "system" fn( + catalog: *const flCatalog, + out_models: *mut *mut flModelList, + ) -> flStatusPtr, + pub GetLoadedModels: unsafe extern "system" fn( + catalog: *const flCatalog, + out_models: *mut *mut flModelList, + ) -> flStatusPtr, +} + +/// Model API table (`flModelApi`). +#[repr(C)] +pub struct flModelApiVtable { + pub GetInfo: unsafe extern "system" fn( + model: *const flModel, + out_info: *mut *const flModelInfo, + ) -> flStatusPtr, + pub GetInputOutputInfo: unsafe extern "system" fn( + model: *const flModel, + out_inputs: *mut *const *const flItem, + out_num_inputs: *mut usize, + out_outputs: *mut *const *const flItem, + out_num_outputs: *mut usize, + ) -> flStatusPtr, + + pub IsCached: + unsafe extern "system" fn(model: *const flModel, out_cached: *mut c_int) -> flStatusPtr, + pub GetPath: unsafe extern "system" fn( + model: *const flModel, + out_path: *mut *const c_char, + ) -> flStatusPtr, + pub Download: unsafe extern "system" fn( + model: *mut flModel, + callback: flProgressCallback, + user_data: *mut c_void, + ) -> flStatusPtr, + + pub IsLoaded: + unsafe extern "system" fn(model: *const flModel, out_loaded: *mut c_int) -> flStatusPtr, + pub Load: unsafe extern "system" fn(model: *mut flModel) -> flStatusPtr, + pub Unload: unsafe extern "system" fn(model: *mut flModel) -> flStatusPtr, + pub RemoveFromCache: unsafe extern "system" fn(model: *mut flModel) -> flStatusPtr, + + pub GetVariants: unsafe extern "system" fn( + model: *const flModel, + out_variants: *mut *mut flModelList, + ) -> flStatusPtr, + pub SelectVariant: + unsafe extern "system" fn(model: *mut flModel, variant: *const flModel) -> flStatusPtr, + + pub Info_GetId: unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetName: unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetVersion: unsafe extern "system" fn(info: *const flModelInfo) -> c_int, + pub Info_GetAlias: unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetUri: unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetDeviceType: unsafe extern "system" fn(info: *const flModelInfo) -> flDeviceType, + pub Info_GetExecutionProvider: + unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetTask: unsafe extern "system" fn(info: *const flModelInfo) -> *const c_char, + pub Info_GetPromptTemplates: + unsafe extern "system" fn(info: *const flModelInfo) -> *const flKeyValuePairs, + pub Info_GetModelSettings: + unsafe extern "system" fn(info: *const flModelInfo) -> *const flKeyValuePairs, + pub Info_GetStringProperty: + unsafe extern "system" fn(info: *const flModelInfo, key: *const c_char) -> *const c_char, + pub Info_GetIntProperty: unsafe extern "system" fn( + info: *const flModelInfo, + key: *const c_char, + default_value: i64, + ) -> i64, +} diff --git a/sdk_v2/rust/src/detail/info.rs b/sdk_v2/rust/src/detail/info.rs new file mode 100644 index 000000000..3300a299e --- /dev/null +++ b/sdk_v2/rust/src/detail/info.rs @@ -0,0 +1,152 @@ +//! Builds the public [`ModelInfo`] from native `flModelInfo` accessors. + +use super::api::{cstr_to_string, read_kvps, Api}; +use super::ffi::*; +use super::native::NativeModel; +use crate::error::Result; +use crate::types::{DeviceType, ModelInfo, ModelSettings, Parameter, PromptTemplate, Runtime}; + +fn device_type(value: flDeviceType) -> DeviceType { + match value { + FOUNDRY_LOCAL_DEVICE_CPU => DeviceType::CPU, + FOUNDRY_LOCAL_DEVICE_GPU => DeviceType::GPU, + FOUNDRY_LOCAL_DEVICE_NPU => DeviceType::NPU, + _ => DeviceType::Invalid, + } +} + +/// Build a fully-populated [`ModelInfo`] for the given model handle. +pub(crate) fn build_model_info(api: &Api, native: &NativeModel) -> Result { + let info = native.info_ptr()?; + let m = api.model_api(); + + // SAFETY: `info` is a valid model-owned info pointer; all accessors below + // return pointers owned by the model and valid for the duration of use. + unsafe { + let id = cstr_to_string((m.Info_GetId)(info)).unwrap_or_default(); + let name = cstr_to_string((m.Info_GetName)(info)).unwrap_or_default(); + let version = (m.Info_GetVersion)(info).max(0) as u64; + let alias = cstr_to_string((m.Info_GetAlias)(info)).unwrap_or_default(); + let uri = cstr_to_string((m.Info_GetUri)(info)).unwrap_or_default(); + let task = cstr_to_string((m.Info_GetTask)(info)); + let execution_provider = cstr_to_string((m.Info_GetExecutionProvider)(info)); + let device = device_type((m.Info_GetDeviceType)(info)); + + let str_prop = |key: &str| -> Option { + let c = std::ffi::CString::new(key).ok()?; + cstr_to_string((m.Info_GetStringProperty)(info, c.as_ptr())) + }; + let int_prop = |key: &str, default_value: i64| -> i64 { + match std::ffi::CString::new(key) { + Ok(c) => (m.Info_GetIntProperty)(info, c.as_ptr(), default_value), + Err(_) => default_value, + } + }; + let opt_u64 = |v: i64| -> Option { + if v >= 0 { + Some(v as u64) + } else { + None + } + }; + let opt_bool = |v: i64| -> Option { + match v { + 0 => Some(false), + 1 => Some(true), + _ => None, + } + }; + + let runtime = execution_provider.clone().map(|ep| Runtime { + device_type: device.clone(), + execution_provider: ep, + }); + + let prompt_template = build_prompt_template(api, (m.Info_GetPromptTemplates)(info)); + let model_settings = build_model_settings(api, (m.Info_GetModelSettings)(info)); + + Ok(ModelInfo { + id, + name, + version, + alias, + display_name: str_prop(FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR), + provider_type: str_prop(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR) + .unwrap_or_default(), + uri, + model_type: str_prop(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR).unwrap_or_default(), + prompt_template, + publisher: str_prop(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR), + model_settings, + license: str_prop(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_STR), + license_description: str_prop(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_DESCRIPTION_STR), + cached: native.is_cached()?, + task, + runtime, + file_size_mb: opt_u64(int_prop(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, -1)), + supports_tool_calling: opt_bool(int_prop( + FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT, + -1, + )), + max_output_tokens: opt_u64(int_prop( + FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT, + -1, + )), + min_fl_version: str_prop(FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR), + created_at_unix: int_prop(FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT, 0).max(0) + as u64, + context_length: opt_u64(int_prop(FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, -1)), + input_modalities: str_prop(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR), + output_modalities: str_prop(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR), + capabilities: str_prop(FOUNDRY_LOCAL_MODEL_PROP_CAPABILITIES_STR), + }) + } +} + +unsafe fn build_prompt_template(api: &Api, kvps: *const flKeyValuePairs) -> Option { + if kvps.is_null() { + return None; + } + let pairs = read_kvps(api, kvps); + if pairs.is_empty() { + return None; + } + let get = |key: &str| -> Option { + pairs + .iter() + .find(|(k, _)| k == key) + .and_then(|(_, v)| v.clone()) + }; + let template = PromptTemplate { + system: get("system"), + user: get("user"), + assistant: get("assistant"), + prompt: get("prompt"), + }; + if template.system.is_none() + && template.user.is_none() + && template.assistant.is_none() + && template.prompt.is_none() + { + None + } else { + Some(template) + } +} + +unsafe fn build_model_settings(api: &Api, kvps: *const flKeyValuePairs) -> Option { + if kvps.is_null() { + return None; + } + let pairs = read_kvps(api, kvps); + if pairs.is_empty() { + return None; + } + let parameters = pairs + .into_iter() + .map(|(name, value)| Parameter { name, value }) + .collect::>(); + Some(ModelSettings { + parameters: Some(parameters), + }) +} diff --git a/sdk_v2/rust/src/detail/items.rs b/sdk_v2/rust/src/detail/items.rs new file mode 100644 index 000000000..42bd975e1 --- /dev/null +++ b/sdk_v2/rust/src/detail/items.rs @@ -0,0 +1,957 @@ +//! Helpers for constructing and reading native `flItem`s. +//! +//! The OpenAI facade and live-audio session build TEXT (`OPENAI_JSON`), AUDIO, +//! and BYTES items, and read TEXT items back out of responses / streamed queues. + +use std::ffi::c_void; +use std::os::raw::c_char; +use std::ptr; + +use super::api::{cstr_to_string, to_cstring, Api}; +use super::ffi::*; +use crate::error::{FoundryLocalError, Result}; +use crate::item::{ + Audio, Image, Item, MediaSource, Message, MessageRole, SpeechResult, SpeechSegment, + SpeechSegmentKind, SpeechWord, Tensor, TensorDataType, TextKind, ToolCall, ToolResult, +}; + +/// Create a TEXT item with the given subtype. The native layer copies the text. +pub(crate) fn make_text_item( + api: &Api, + text: &str, + item_type: flTextItemType, +) -> Result<*mut flItem> { + // Convert before Create so a NUL-conversion error can't leak the item. + let c = to_cstring(text)?; + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_TEXT, &mut item) })?; + let data = flTextData { + version: FOUNDRY_LOCAL_API_VERSION, + text: c.as_ptr(), + r#type: item_type, + }; + // SAFETY: `item` is a valid TEXT item; the native call copies the string. + let status = unsafe { (api.item_api().SetText)(item, &data) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + Ok(item) +} + +/// Create a TEXT item carrying an opaque OpenAI REST JSON payload. +pub(crate) fn make_openai_json_item(api: &Api, json: &str) -> Result<*mut flItem> { + make_text_item(api, json, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON) +} + +/// Create a byte-backed AUDIO item describing a PCM format (used as the live +/// audio format descriptor) or carrying audio bytes. The native layer copies +/// the data, so the caller's buffer need not outlive the call. +pub(crate) fn make_audio_item( + api: &Api, + data: &[u8], + format: Option<&str>, + sample_rate: i32, + channels: i32, +) -> Result<*mut flItem> { + // Convert before Create so a NUL-conversion error can't leak the item. + let format_c = match format { + Some(f) => Some(to_cstring(f)?), + None => None, + }; + + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_AUDIO, &mut item) })?; + + // Like SetBytes, SetAudio does not copy the sample buffer — it borrows the + // pointer (and frees it via the deleter when one is supplied). Transfer an + // owned heap allocation so the buffer outlives this call; the format string + // is copied natively, so a transient CString is fine. + let (data_ptr, len, deleter): (*mut u8, usize, flAudioDataDeleter) = if data.is_empty() { + (ptr::null_mut(), 0, None) + } else { + let boxed: Box<[u8]> = data.to_vec().into_boxed_slice(); + let len = boxed.len(); + ( + Box::into_raw(boxed) as *mut u8, + len, + Some(rust_audio_deleter), + ) + }; + + let audio = flAudioData { + version: FOUNDRY_LOCAL_API_VERSION, + data: data_ptr as *const std::ffi::c_void, + mutable_data: data_ptr as *mut std::ffi::c_void, + data_size: len, + format: format_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + uri: ptr::null(), + sample_rate, + channels, + deleter, + deleter_user_data: ptr::null_mut(), + }; + // SAFETY: `item` is a valid AUDIO item. On success the item owns `data_ptr` + // (freed via the deleter); on failure we reclaim it here to avoid a leak. + let status = unsafe { (api.item_api().SetAudio)(item, &audio) }; + if let Err(e) = api.check(status) { + unsafe { + if !data_ptr.is_null() { + drop(Box::from_raw(ptr::slice_from_raw_parts_mut(data_ptr, len))); + } + (api.item_api().Item_Release)(item); + } + return Err(e); + } + Ok(item) +} + +/// Deleter that reclaims a Rust-allocated `Box<[u8]>` owned by an AUDIO item. +/// Mirrors [`rust_bytes_deleter`]; see its docs for the ownership contract. +unsafe extern "C" fn rust_audio_deleter( + data: *const flAudioData, + _user_data: *mut std::ffi::c_void, +) { + if data.is_null() { + return; + } + let d = &*data; + if !d.mutable_data.is_null() && d.data_size > 0 { + let slice = ptr::slice_from_raw_parts_mut(d.mutable_data as *mut u8, d.data_size); + drop(Box::from_raw(slice)); + } +} + +/// Deleter that reclaims a Rust-allocated `Box<[u8]>` owned by a BYTES item. +/// +/// The native item calls this on destruction. `mutable_data` is the pointer we +/// handed over via `Box::into_raw`, and `data_size` is its length; together they +/// reconstruct the boxed slice so it is dropped exactly once. +unsafe extern "C" fn rust_bytes_deleter( + data: *const flBytesData, + _user_data: *mut std::ffi::c_void, +) { + if data.is_null() { + return; + } + let d = &*data; + if !d.mutable_data.is_null() && d.data_size > 0 { + let slice = ptr::slice_from_raw_parts_mut(d.mutable_data as *mut u8, d.data_size); + drop(Box::from_raw(slice)); + } +} + +/// Create a BYTES item tagged with the given originating item type (e.g. AUDIO +/// for raw PCM chunks pushed into a live session). +/// +/// The native `SetBytes` does **not** copy — it stores the pointer and (when a +/// deleter is supplied) takes ownership of the buffer, freeing it via the +/// deleter when the item is destroyed. The item may be consumed asynchronously +/// (e.g. drained from an `ItemQueue` by a streaming worker long after this +/// returns), so the buffer must outlive this call. We therefore transfer an +/// owned heap allocation to the item rather than lending a caller buffer. +pub(crate) fn make_bytes_item( + api: &Api, + data: &[u8], + item_type: flItemType, +) -> Result<*mut flItem> { + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_BYTES, &mut item) })?; + + if data.is_empty() { + let bytes = flBytesData { + version: FOUNDRY_LOCAL_API_VERSION, + item_type, + data: ptr::null(), + mutable_data: ptr::null_mut(), + data_size: 0, + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + let status = unsafe { (api.item_api().SetBytes)(item, &bytes) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + return Ok(item); + } + + // Transfer an owned copy to the item. `into_boxed_slice` guarantees + // capacity == len, so the deleter can reconstruct it from (ptr, len). + let boxed: Box<[u8]> = data.to_vec().into_boxed_slice(); + let len = boxed.len(); + let raw = Box::into_raw(boxed) as *mut u8; + + let bytes = flBytesData { + version: FOUNDRY_LOCAL_API_VERSION, + item_type, + data: raw as *const std::ffi::c_void, + mutable_data: raw as *mut std::ffi::c_void, + data_size: len, + deleter: Some(rust_bytes_deleter), + deleter_user_data: ptr::null_mut(), + }; + + // SAFETY: `item` is a valid BYTES item. On success the item owns `raw` and + // frees it via the deleter; on failure SetBytesData was not applied, so we + // reclaim and drop the box here to avoid leaking it. + let status = unsafe { (api.item_api().SetBytes)(item, &bytes) }; + if let Err(e) = api.check(status) { + unsafe { + let slice = ptr::slice_from_raw_parts_mut(raw, len); + drop(Box::from_raw(slice)); + (api.item_api().Item_Release)(item); + } + return Err(e); + } + Ok(item) +} + +/// Read the text of a TEXT item. Returns `None` for null/non-text items. +/// +/// # Safety +/// `item` must be null or a valid item pointer alive for the duration of this call. +pub(crate) unsafe fn read_text_item(api: &Api, item: *const flItem) -> Option { + if item.is_null() { + return None; + } + if (api.item_api().GetType)(item) != FOUNDRY_LOCAL_ITEM_TEXT { + return None; + } + let mut data = flTextData { + version: FOUNDRY_LOCAL_API_VERSION, + text: ptr::null::(), + r#type: FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT, + }; + if api + .check((api.item_api().GetText)(item, &mut data)) + .is_err() + { + return None; + } + cstr_to_string(data.text) +} + +/// Text + timing read from a SPEECH_SEGMENT item (output-only). +pub(crate) struct SpeechSegmentText { + pub text: String, + pub is_final: bool, + pub start_time_s: Option, + pub end_time_s: Option, +} + +/// Convert a native millisecond field to seconds, mapping the UNSET sentinel to `None`. +fn duration_ms_to_seconds(ms: i64) -> Option { + if ms == FOUNDRY_LOCAL_DURATION_UNSET { + None + } else { + Some(ms as f64 / 1000.0) + } +} + +/// Read a SPEECH_SEGMENT item (output-only). Returns `None` for null/other items. +/// +/// # Safety +/// `item` must be null or a valid item pointer alive for the duration of this call. +pub(crate) unsafe fn read_speech_segment( + api: &Api, + item: *const flItem, +) -> Option { + if item.is_null() || (api.item_api().GetType)(item) != FOUNDRY_LOCAL_ITEM_SPEECH_SEGMENT { + return None; + } + let mut data = flSpeechSegmentData { + version: FOUNDRY_LOCAL_API_VERSION, + kind: FOUNDRY_LOCAL_SPEECH_SEGMENT_NONE, + text: ptr::null::(), + start_time_ms: FOUNDRY_LOCAL_DURATION_UNSET, + end_time_ms: FOUNDRY_LOCAL_DURATION_UNSET, + utterance_start: false, + words: ptr::null::(), + words_count: 0, + language: ptr::null::(), + }; + if api + .check((api.item_api().GetSpeechSegment)(item, &mut data)) + .is_err() + { + return None; + } + Some(SpeechSegmentText { + text: cstr_to_string(data.text).unwrap_or_default(), + // PARTIAL is an interim hypothesis; FINAL (and NONE entries) are stable. + is_final: data.kind == FOUNDRY_LOCAL_SPEECH_SEGMENT_FINAL, + start_time_s: duration_ms_to_seconds(data.start_time_ms), + end_time_s: duration_ms_to_seconds(data.end_time_ms), + }) +} + +/// Read the concatenated transcript of a SPEECH_RESULT item (output-only). +/// Returns `None` for null/other items. +/// +/// # Safety +/// `item` must be null or a valid item pointer alive for the duration of this call. +pub(crate) unsafe fn read_speech_result_text(api: &Api, item: *const flItem) -> Option { + if item.is_null() || (api.item_api().GetType)(item) != FOUNDRY_LOCAL_ITEM_SPEECH_RESULT { + return None; + } + let mut data = flSpeechResultData { + version: FOUNDRY_LOCAL_API_VERSION, + text: ptr::null::(), + language: ptr::null::(), + duration_ms: FOUNDRY_LOCAL_DURATION_UNSET, + segments: ptr::null::<*const flItem>(), + segments_count: 0, + }; + if api + .check((api.item_api().GetSpeechResult)(item, &mut data)) + .is_err() + { + return None; + } + cstr_to_string(data.text) +} + +// ── Additional native item builders (image / tensor / message / tool) ───────── + +/// Deleter reclaiming a Rust-allocated `Box<[u8]>` owned by an IMAGE item. +/// Mirrors [`rust_bytes_deleter`]; `data_size` gives the boxed slice length. +unsafe extern "C" fn rust_image_deleter(data: *const flImageData, _user_data: *mut c_void) { + if data.is_null() { + return; + } + let d = &*data; + if !d.mutable_data.is_null() && d.data_size > 0 { + let slice = ptr::slice_from_raw_parts_mut(d.mutable_data as *mut u8, d.data_size); + drop(Box::from_raw(slice)); + } +} + +/// Deleter reclaiming a Rust-allocated `Box<[u8]>` owned by a TENSOR item. +/// +/// Unlike bytes/image/audio, `flTensorData` carries no `data_size`, so the byte +/// length is smuggled through `deleter_user_data` as a boxed `usize`. Both the +/// data box and the length box are reclaimed here (each exactly once). +unsafe extern "C" fn rust_tensor_deleter(data: *const flTensorData, user_data: *mut c_void) { + let len = if user_data.is_null() { + 0 + } else { + *Box::from_raw(user_data as *mut usize) + }; + if data.is_null() { + return; + } + let d = &*data; + if !d.mutable_data.is_null() && len > 0 { + let slice = ptr::slice_from_raw_parts_mut(d.mutable_data as *mut u8, len); + drop(Box::from_raw(slice)); + } +} + +/// Create a TENSOR item. The native layer copies the shape but borrows (and, via +/// the deleter, takes ownership of) the element buffer, so we transfer an owned +/// heap allocation whose length travels with the deleter's user data. +pub(crate) fn make_tensor_item(api: &Api, tensor: &Tensor) -> Result<*mut flItem> { + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_TENSOR, &mut item) })?; + + // `shape` only needs to outlive the SetTensor call (native copies it). + let shape = tensor.shape.clone(); + + let (data_ptr, len, deleter, user_data): (*mut u8, usize, flTensorDataDeleter, *mut c_void) = + if tensor.data.is_empty() { + (ptr::null_mut(), 0, None, ptr::null_mut()) + } else { + let boxed: Box<[u8]> = tensor.data.clone().into_boxed_slice(); + let len = boxed.len(); + let raw = Box::into_raw(boxed) as *mut u8; + let len_box = Box::into_raw(Box::new(len)) as *mut c_void; + (raw, len, Some(rust_tensor_deleter), len_box) + }; + + let data = flTensorData { + version: FOUNDRY_LOCAL_API_VERSION, + data_type: tensor.data_type.to_native(), + data: data_ptr as *const c_void, + mutable_data: data_ptr as *mut c_void, + shape: shape.as_ptr(), + rank: shape.len(), + deleter, + deleter_user_data: user_data, + }; + // SAFETY: `item` is a valid TENSOR item. On success it owns `data_ptr` (freed + // via the deleter); on failure we reclaim both the data and length boxes. + let status = unsafe { (api.item_api().SetTensor)(item, &data) }; + if let Err(e) = api.check(status) { + unsafe { + if !data_ptr.is_null() { + drop(Box::from_raw(ptr::slice_from_raw_parts_mut(data_ptr, len))); + } + if !user_data.is_null() { + drop(Box::from_raw(user_data as *mut usize)); + } + (api.item_api().Item_Release)(item); + } + return Err(e); + } + Ok(item) +} + +/// Create an IMAGE item from inline bytes or a URI. Native copies the format/URI +/// strings; inline bytes are transferred as an owned heap allocation. +pub(crate) fn make_image_item(api: &Api, image: &Image) -> Result<*mut flItem> { + let format_c = match &image.format { + Some(f) => Some(to_cstring(f)?), + None => None, + }; + let uri_c = match &image.source { + MediaSource::Uri(u) => Some(to_cstring(u)?), + MediaSource::Data(_) => None, + }; + + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_IMAGE, &mut item) })?; + + let (data_ptr, len, deleter): (*mut u8, usize, flImageDataDeleter) = match &image.source { + MediaSource::Data(bytes) if !bytes.is_empty() => { + let boxed: Box<[u8]> = bytes.clone().into_boxed_slice(); + let len = boxed.len(); + ( + Box::into_raw(boxed) as *mut u8, + len, + Some(rust_image_deleter), + ) + } + _ => (ptr::null_mut(), 0, None), + }; + + let data = flImageData { + version: FOUNDRY_LOCAL_API_VERSION, + data: data_ptr as *const c_void, + mutable_data: data_ptr as *mut c_void, + data_size: len, + format: format_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + uri: uri_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + deleter, + deleter_user_data: ptr::null_mut(), + }; + // SAFETY: `item` is a valid IMAGE item; on failure we reclaim the data box. + let status = unsafe { (api.item_api().SetImage)(item, &data) }; + if let Err(e) = api.check(status) { + unsafe { + if !data_ptr.is_null() { + drop(Box::from_raw(ptr::slice_from_raw_parts_mut(data_ptr, len))); + } + (api.item_api().Item_Release)(item); + } + return Err(e); + } + Ok(item) +} + +/// Create an AUDIO item that references external content by URI (no sample data). +pub(crate) fn make_audio_uri_item( + api: &Api, + uri: &str, + format: Option<&str>, + sample_rate: i32, + channels: i32, +) -> Result<*mut flItem> { + let uri_c = to_cstring(uri)?; + let format_c = match format { + Some(f) => Some(to_cstring(f)?), + None => None, + }; + + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_AUDIO, &mut item) })?; + + let audio = flAudioData { + version: FOUNDRY_LOCAL_API_VERSION, + data: ptr::null(), + mutable_data: ptr::null_mut(), + data_size: 0, + format: format_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + uri: uri_c.as_ptr(), + sample_rate, + channels, + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + let status = unsafe { (api.item_api().SetAudio)(item, &audio) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + Ok(item) +} + +/// Create a TOOL_CALL item. Native copies all three strings. +pub(crate) fn make_tool_call_item(api: &Api, tool_call: &ToolCall) -> Result<*mut flItem> { + let call_id = to_cstring(&tool_call.call_id)?; + let name = to_cstring(&tool_call.name)?; + let arguments = to_cstring(&tool_call.arguments)?; + + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_TOOL_CALL, &mut item) })?; + let data = flToolCallData { + version: FOUNDRY_LOCAL_API_VERSION, + call_id: call_id.as_ptr(), + name: name.as_ptr(), + arguments: arguments.as_ptr(), + }; + let status = unsafe { (api.item_api().SetToolCall)(item, &data) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + Ok(item) +} + +/// Create a TOOL_RESULT item. Native copies both strings. +pub(crate) fn make_tool_result_item(api: &Api, tool_result: &ToolResult) -> Result<*mut flItem> { + let call_id = to_cstring(&tool_result.call_id)?; + let result = to_cstring(&tool_result.result)?; + + let mut item: *mut flItem = ptr::null_mut(); + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_TOOL_RESULT, &mut item) })?; + let data = flToolResultData { + version: FOUNDRY_LOCAL_API_VERSION, + call_id: call_id.as_ptr(), + result: result.as_ptr(), + }; + let status = unsafe { (api.item_api().SetToolResult)(item, &data) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + Ok(item) +} + +/// Release a batch of transient native items (used to unwind on error). +unsafe fn release_all(api: &Api, items: &[*mut flItem]) { + for &it in items { + if !it.is_null() { + (api.item_api().Item_Release)(it); + } + } +} + +/// Create a MESSAGE item. Its content parts are built as transient native items, +/// handed to `SetMessage` (which **deep-clones** them), then released. Only +/// TEXT/IMAGE/AUDIO parts are accepted by the native layer. +pub(crate) fn make_message_item(api: &Api, message: &Message) -> Result<*mut flItem> { + // Build children first so a failure can't leave a half-populated message. + let mut children: Vec<*mut flItem> = Vec::with_capacity(message.content.len()); + for part in &message.content { + match item_to_native(api, part) { + Ok(child) => children.push(child), + Err(e) => { + unsafe { release_all(api, &children) }; + return Err(e); + } + } + } + + let name_c = match &message.name { + Some(n) => match to_cstring(n) { + Ok(c) => Some(c), + Err(e) => { + unsafe { release_all(api, &children) }; + return Err(e); + } + }, + None => None, + }; + + let mut item: *mut flItem = ptr::null_mut(); + if let Err(e) = + api.check(unsafe { (api.item_api().Create)(FOUNDRY_LOCAL_ITEM_MESSAGE, &mut item) }) + { + unsafe { release_all(api, &children) }; + return Err(e); + } + + let child_ptrs: Vec<*const flItem> = children.iter().map(|p| *p as *const flItem).collect(); + let data = flMessageData { + version: FOUNDRY_LOCAL_API_VERSION, + role: message.role.to_native(), + content_items: if child_ptrs.is_empty() { + ptr::null() + } else { + child_ptrs.as_ptr() + }, + content_items_count: child_ptrs.len(), + name: name_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + }; + let status = unsafe { (api.item_api().SetMessage)(item, &data) }; + // Native deep-clones each part on success; on failure it retains nothing. + // Either way the transient children are ours to release now. + unsafe { release_all(api, &children) }; + if let Err(e) = api.check(status) { + unsafe { (api.item_api().Item_Release)(item) }; + return Err(e); + } + Ok(item) +} + +/// Build a native `flItem` from a public [`Item`]. The returned pointer is owned +/// by the caller (release via `Item_Release`, or transfer into a request/queue). +/// +/// Speech items are output-only and rejected with a validation error. +pub(crate) fn item_to_native(api: &Api, item: &Item) -> Result<*mut flItem> { + match item { + Item::Text { text, kind } => make_text_item(api, text, kind.to_native()), + Item::Message(m) => make_message_item(api, m), + Item::Bytes(data) => make_bytes_item(api, data, FOUNDRY_LOCAL_ITEM_BYTES), + Item::Tensor(t) => make_tensor_item(api, t), + Item::Image(img) => make_image_item(api, img), + Item::Audio(a) => match &a.source { + MediaSource::Data(d) => { + make_audio_item(api, d, a.format.as_deref(), a.sample_rate, a.channels) + } + MediaSource::Uri(u) => { + make_audio_uri_item(api, u, a.format.as_deref(), a.sample_rate, a.channels) + } + }, + Item::ToolCall(c) => make_tool_call_item(api, c), + Item::ToolResult(r) => make_tool_result_item(api, r), + Item::SpeechSegment(_) | Item::SpeechResult(_) => Err(FoundryLocalError::Validation { + reason: "speech items are output-only and cannot be added to a request".into(), + }), + } +} + +// ── Full native item readers (native `flItem` → public `Item`) ──────────────── + +/// Copy `size` bytes from a borrowed native buffer into an owned `Vec`. +/// +/// # Safety +/// `data`, when non-null, must point to at least `size` valid bytes. +unsafe fn copy_bytes(data: *const c_void, size: usize) -> Vec { + if data.is_null() || size == 0 { + Vec::new() + } else { + std::slice::from_raw_parts(data as *const u8, size).to_vec() + } +} + +/// Map a native millisecond field to `Option`, treating the UNSET sentinel as `None`. +fn opt_ms(ms: i64) -> Option { + if ms == FOUNDRY_LOCAL_DURATION_UNSET { + None + } else { + Some(ms) + } +} + +/// The number of bytes backing a tensor of `data_type` and `shape`, or `None` +/// when the element size is not statically known (e.g. STRING tensors). +fn tensor_byte_len(data_type: TensorDataType, shape: &[i64]) -> Option { + let bits: usize = match data_type { + TensorDataType::Uint8 + | TensorDataType::Int8 + | TensorDataType::Bool + | TensorDataType::Float8E4M3FN + | TensorDataType::Float8E4M3FNUZ + | TensorDataType::Float8E5M2 + | TensorDataType::Float8E5M2FNUZ + | TensorDataType::Float8E8M0 => 8, + TensorDataType::Uint16 + | TensorDataType::Int16 + | TensorDataType::Float16 + | TensorDataType::BFloat16 => 16, + TensorDataType::Float | TensorDataType::Int32 | TensorDataType::Uint32 => 32, + TensorDataType::Int64 + | TensorDataType::Uint64 + | TensorDataType::Double + | TensorDataType::Complex64 => 64, + TensorDataType::Complex128 => 128, + TensorDataType::Uint4 | TensorDataType::Int4 | TensorDataType::Float4E2M1 => 4, + TensorDataType::String | TensorDataType::Undefined => return None, + }; + let mut elements: i128 = 1; + for &dim in shape { + if dim < 0 { + return None; + } + elements = elements.checked_mul(dim as i128)?; + } + let total_bits = elements.checked_mul(bits as i128)?; + usize::try_from((total_bits + 7) / 8).ok() +} + +/// Read a TEXT item into an owned [`Item::Text`] (preserving its [`TextKind`]). +unsafe fn read_text_full(api: &Api, item: *const flItem) -> Option { + let mut data = flTextData { + version: FOUNDRY_LOCAL_API_VERSION, + text: ptr::null::(), + r#type: FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT, + }; + api.check((api.item_api().GetText)(item, &mut data)).ok()?; + Some(Item::Text { + text: cstr_to_string(data.text).unwrap_or_default(), + kind: TextKind::from_native(data.r#type), + }) +} + +/// Read a BYTES item into an owned [`Item::Bytes`]. +unsafe fn read_bytes_full(api: &Api, item: *const flItem) -> Option { + let mut data = flBytesData { + version: FOUNDRY_LOCAL_API_VERSION, + item_type: FOUNDRY_LOCAL_ITEM_BYTES, + data: ptr::null(), + mutable_data: ptr::null_mut(), + data_size: 0, + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + api.check((api.item_api().GetBytes)(item, &mut data)).ok()?; + Some(Item::Bytes(copy_bytes(data.data, data.data_size))) +} + +/// Read a TENSOR item into an owned [`Item::Tensor`]. +unsafe fn read_tensor_full(api: &Api, item: *const flItem) -> Option { + let mut data = flTensorData { + version: FOUNDRY_LOCAL_API_VERSION, + data_type: FOUNDRY_LOCAL_TENSOR_UNDEFINED, + data: ptr::null(), + mutable_data: ptr::null_mut(), + shape: ptr::null(), + rank: 0, + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + api.check((api.item_api().GetTensor)(item, &mut data)) + .ok()?; + let shape: Vec = if data.shape.is_null() || data.rank == 0 { + Vec::new() + } else { + std::slice::from_raw_parts(data.shape, data.rank).to_vec() + }; + let data_type = TensorDataType::from_native(data.data_type); + let bytes = match tensor_byte_len(data_type, &shape) { + Some(len) => copy_bytes(data.data, len), + None => Vec::new(), + }; + Some(Item::Tensor(Tensor { + data_type, + shape, + data: bytes, + })) +} + +/// Read an IMAGE item into an owned [`Item::Image`]. +unsafe fn read_image_full(api: &Api, item: *const flItem) -> Option { + let mut data = flImageData { + version: FOUNDRY_LOCAL_API_VERSION, + data: ptr::null(), + mutable_data: ptr::null_mut(), + data_size: 0, + format: ptr::null(), + uri: ptr::null(), + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + api.check((api.item_api().GetImage)(item, &mut data)).ok()?; + let source = if !data.data.is_null() && data.data_size > 0 { + MediaSource::Data(copy_bytes(data.data, data.data_size)) + } else if let Some(uri) = cstr_to_string(data.uri) { + MediaSource::Uri(uri) + } else { + MediaSource::Data(Vec::new()) + }; + Some(Item::Image(Image { + source, + format: cstr_to_string(data.format), + })) +} + +/// Read an AUDIO item into an owned [`Item::Audio`]. +unsafe fn read_audio_full(api: &Api, item: *const flItem) -> Option { + let mut data = flAudioData { + version: FOUNDRY_LOCAL_API_VERSION, + data: ptr::null(), + mutable_data: ptr::null_mut(), + data_size: 0, + format: ptr::null(), + uri: ptr::null(), + sample_rate: 0, + channels: 0, + deleter: None, + deleter_user_data: ptr::null_mut(), + }; + api.check((api.item_api().GetAudio)(item, &mut data)).ok()?; + let source = if !data.data.is_null() && data.data_size > 0 { + MediaSource::Data(copy_bytes(data.data, data.data_size)) + } else if let Some(uri) = cstr_to_string(data.uri) { + MediaSource::Uri(uri) + } else { + MediaSource::Data(Vec::new()) + }; + Some(Item::Audio(Audio { + source, + format: cstr_to_string(data.format), + sample_rate: data.sample_rate, + channels: data.channels, + })) +} + +/// Read a MESSAGE item into an owned [`Item::Message`], recursively decoding parts. +unsafe fn read_message_full(api: &Api, item: *const flItem) -> Option { + let mut data = flMessageData { + version: FOUNDRY_LOCAL_API_VERSION, + role: FOUNDRY_LOCAL_ROLE_NONE, + content_items: ptr::null(), + content_items_count: 0, + name: ptr::null(), + }; + api.check((api.item_api().GetMessage)(item, &mut data)) + .ok()?; + let mut content = Vec::with_capacity(data.content_items_count); + if !data.content_items.is_null() { + for i in 0..data.content_items_count { + let part = *data.content_items.add(i); + if let Some(child) = item_from_native(api, part) { + content.push(child); + } + } + } + Some(Item::Message(Message { + role: MessageRole::from_native(data.role), + content, + name: cstr_to_string(data.name), + })) +} + +/// Read a TOOL_CALL item into an owned [`Item::ToolCall`]. +unsafe fn read_tool_call_full(api: &Api, item: *const flItem) -> Option { + let mut data = flToolCallData { + version: FOUNDRY_LOCAL_API_VERSION, + call_id: ptr::null(), + name: ptr::null(), + arguments: ptr::null(), + }; + api.check((api.item_api().GetToolCall)(item, &mut data)) + .ok()?; + Some(Item::ToolCall(ToolCall { + call_id: cstr_to_string(data.call_id).unwrap_or_default(), + name: cstr_to_string(data.name).unwrap_or_default(), + arguments: cstr_to_string(data.arguments).unwrap_or_default(), + })) +} + +/// Read a TOOL_RESULT item into an owned [`Item::ToolResult`]. +unsafe fn read_tool_result_full(api: &Api, item: *const flItem) -> Option { + let mut data = flToolResultData { + version: FOUNDRY_LOCAL_API_VERSION, + call_id: ptr::null(), + result: ptr::null(), + }; + api.check((api.item_api().GetToolResult)(item, &mut data)) + .ok()?; + Some(Item::ToolResult(ToolResult { + call_id: cstr_to_string(data.call_id).unwrap_or_default(), + result: cstr_to_string(data.result).unwrap_or_default(), + })) +} + +/// Read a SPEECH_SEGMENT item into an owned [`SpeechSegment`] value. +unsafe fn read_speech_segment_value(api: &Api, item: *const flItem) -> Option { + let mut data = flSpeechSegmentData { + version: FOUNDRY_LOCAL_API_VERSION, + kind: FOUNDRY_LOCAL_SPEECH_SEGMENT_NONE, + text: ptr::null::(), + start_time_ms: FOUNDRY_LOCAL_DURATION_UNSET, + end_time_ms: FOUNDRY_LOCAL_DURATION_UNSET, + utterance_start: false, + words: ptr::null::(), + words_count: 0, + language: ptr::null::(), + }; + api.check((api.item_api().GetSpeechSegment)(item, &mut data)) + .ok()?; + let mut words = Vec::with_capacity(data.words_count); + if !data.words.is_null() { + for i in 0..data.words_count { + let w = &*data.words.add(i); + let confidence = if w.confidence == FOUNDRY_LOCAL_CONFIDENCE_UNSET { + None + } else { + Some(w.confidence) + }; + words.push(SpeechWord { + text: cstr_to_string(w.text).unwrap_or_default(), + start_time_ms: opt_ms(w.start_time_ms), + end_time_ms: opt_ms(w.end_time_ms), + confidence, + speaker_id: cstr_to_string(w.speaker_id), + }); + } + } + Some(SpeechSegment { + kind: SpeechSegmentKind::from_native(data.kind), + text: cstr_to_string(data.text).unwrap_or_default(), + start_time_ms: opt_ms(data.start_time_ms), + end_time_ms: opt_ms(data.end_time_ms), + utterance_start: data.utterance_start, + words, + language: cstr_to_string(data.language), + }) +} + +/// Read a SPEECH_RESULT item into an owned [`Item::SpeechResult`]. +unsafe fn read_speech_result_full(api: &Api, item: *const flItem) -> Option { + let mut data = flSpeechResultData { + version: FOUNDRY_LOCAL_API_VERSION, + text: ptr::null::(), + language: ptr::null::(), + duration_ms: FOUNDRY_LOCAL_DURATION_UNSET, + segments: ptr::null::<*const flItem>(), + segments_count: 0, + }; + api.check((api.item_api().GetSpeechResult)(item, &mut data)) + .ok()?; + let mut segments = Vec::with_capacity(data.segments_count); + if !data.segments.is_null() { + for i in 0..data.segments_count { + let seg = *data.segments.add(i); + if let Some(child) = item_from_native(api, seg) { + segments.push(child); + } + } + } + Some(Item::SpeechResult(SpeechResult { + text: cstr_to_string(data.text).unwrap_or_default(), + language: cstr_to_string(data.language), + duration_ms: opt_ms(data.duration_ms), + segments, + })) +} + +/// Decode a native `flItem` into an owned public [`Item`]. +/// +/// Returns `None` for a null item or an item whose type is not representable. +/// +/// # Safety +/// `item` must be null or a valid item pointer alive for the duration of this call. +pub(crate) unsafe fn item_from_native(api: &Api, item: *const flItem) -> Option { + if item.is_null() { + return None; + } + match (api.item_api().GetType)(item) { + FOUNDRY_LOCAL_ITEM_TEXT => read_text_full(api, item), + FOUNDRY_LOCAL_ITEM_BYTES => read_bytes_full(api, item), + FOUNDRY_LOCAL_ITEM_TENSOR => read_tensor_full(api, item), + FOUNDRY_LOCAL_ITEM_MESSAGE => read_message_full(api, item), + FOUNDRY_LOCAL_ITEM_IMAGE => read_image_full(api, item), + FOUNDRY_LOCAL_ITEM_AUDIO => read_audio_full(api, item), + FOUNDRY_LOCAL_ITEM_TOOL_CALL => read_tool_call_full(api, item), + FOUNDRY_LOCAL_ITEM_TOOL_RESULT => read_tool_result_full(api, item), + FOUNDRY_LOCAL_ITEM_SPEECH_SEGMENT => { + read_speech_segment_value(api, item).map(Item::SpeechSegment) + } + FOUNDRY_LOCAL_ITEM_SPEECH_RESULT => read_speech_result_full(api, item), + _ => None, + } +} diff --git a/sdk_v2/rust/src/detail/manager.rs b/sdk_v2/rust/src/detail/manager.rs new file mode 100644 index 000000000..908625095 --- /dev/null +++ b/sdk_v2/rust/src/detail/manager.rs @@ -0,0 +1,255 @@ +//! Owning wrapper around a native `flManager` plus its discovery / web-service / +//! execution-provider operations. + +use std::os::raw::c_char; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use super::api::{cstr_to_string, Api}; +use super::ffi::*; +use crate::error::{FoundryLocalError, Result}; +use crate::types::EpInfo; + +/// Serializes native manager creation against release. +/// +/// The native core enforces a single live `flManager` (`Manager_Create` fails +/// with `INVALID_USAGE` while one exists). Because an [`Arc`]'s +/// strong count reaches zero slightly before [`Drop`] finishes `Manager_Release`, +/// a concurrent `create` could otherwise observe "no instance" yet race the +/// in-flight release and be rejected. Holding this lock across both +/// `Manager_Create` and `Manager_Release` closes that window. +static NATIVE_LIFECYCLE: Mutex<()> = Mutex::new(()); + +/// Lock [`NATIVE_LIFECYCLE`], recovering from poisoning (the guarded `()` has no +/// invariants a panic could break). +fn lock_lifecycle() -> std::sync::MutexGuard<'static, ()> { + NATIVE_LIFECYCLE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Owns a native `flManager`. +/// +/// The native handle is released exactly once by [`Drop`] (via +/// [`teardown`](Self::teardown)) when the last owner is dropped — while the ORT +/// runtime is still alive and before the library's C++ static destructors run. +pub(crate) struct NativeManager { + api: Arc, + ptr: *mut flManager, + /// Set once the native manager has been released, so `shutdown`, explicit + /// `teardown`, and `Drop` all coordinate to release exactly once. + released: AtomicBool, +} + +// SAFETY: the native manager is thread-safe; shutdown is documented as callable +// from any thread. +unsafe impl Send for NativeManager {} +unsafe impl Sync for NativeManager {} + +impl NativeManager { + /// Create a manager from a fully-built native configuration. + pub(crate) fn create(api: Arc, config: *const flConfiguration) -> Result { + // Serialize against any in-flight native release (see `NATIVE_LIFECYCLE`). + let _lifecycle = lock_lifecycle(); + let mut ptr: *mut flManager = std::ptr::null_mut(); + let status = unsafe { (api.root().Manager_Create)(config, &mut ptr) }; + api.check(status)?; + if ptr.is_null() { + return Err(FoundryLocalError::Internal { + reason: "Manager_Create returned a null manager".into(), + }); + } + Ok(Self { + api, + ptr, + released: AtomicBool::new(false), + }) + } + + /// The manager-owned catalog handle. + pub(crate) fn catalog_ptr(&self) -> Result<*mut flCatalog> { + let mut catalog: *mut flCatalog = std::ptr::null_mut(); + let status = unsafe { (self.api.root().Manager_GetCatalog)(self.ptr, &mut catalog) }; + self.api.check(status)?; + if catalog.is_null() { + return Err(FoundryLocalError::Internal { + reason: "Manager_GetCatalog returned a null catalog".into(), + }); + } + Ok(catalog) + } + + pub(crate) fn web_service_start(&self) -> Result<()> { + let status = unsafe { (self.api.root().Manager_WebServiceStart)(self.ptr) }; + self.api.check(status) + } + + pub(crate) fn web_service_stop(&self) -> Result<()> { + let status = unsafe { (self.api.root().Manager_WebServiceStop)(self.ptr) }; + self.api.check(status) + } + + pub(crate) fn web_service_urls(&self) -> Result> { + let mut urls: *const *const c_char = std::ptr::null(); + let mut count: usize = 0; + let status = + unsafe { (self.api.root().Manager_WebServiceUrls)(self.ptr, &mut urls, &mut count) }; + self.api.check(status)?; + if urls.is_null() || count == 0 { + return Ok(Vec::new()); + } + let mut out = Vec::with_capacity(count); + for i in 0..count { + // SAFETY: `urls` points to `count` valid C-string pointers. + if let Some(s) = unsafe { cstr_to_string(*urls.add(i)) } { + out.push(s); + } + } + Ok(out) + } + + pub(crate) fn discover_eps(&self) -> Result> { + let mut eps: *const flEpInfo = std::ptr::null(); + let mut count: usize = 0; + let status = + unsafe { (self.api.root().Manager_GetDiscoverableEps)(self.ptr, &mut eps, &mut count) }; + self.api.check(status)?; + if eps.is_null() || count == 0 { + return Ok(Vec::new()); + } + let mut out = Vec::with_capacity(count); + for i in 0..count { + // SAFETY: `eps` points to `count` valid flEpInfo structs. + let ep = unsafe { &*eps.add(i) }; + out.push(EpInfo { + name: unsafe { cstr_to_string(ep.name) }.unwrap_or_default(), + is_registered: ep.is_registered, + }); + } + Ok(out) + } + + /// Run an EP download/registration. Returns `None` on full success or + /// `Some(message)` describing a partial failure (a non-null native status). + pub(crate) fn download_and_register_eps( + &self, + names: Option<&[&str]>, + progress: Option, + cancel_flag: Option>, + ) -> Option { + // Build the optional C array of name pointers (kept alive across the call). + let name_cstrings: Option> = names.map(|ns| { + ns.iter() + .filter_map(|n| std::ffi::CString::new(*n).ok()) + .collect() + }); + let name_ptrs: Option> = name_cstrings + .as_ref() + .map(|cs| cs.iter().map(|c| c.as_ptr()).collect()); + + let (names_ptr, names_len) = match &name_ptrs { + Some(p) if !p.is_empty() => (p.as_ptr(), p.len()), + _ => (std::ptr::null(), 0usize), + }; + + let mut ctx = EpCtx { + progress, + cancel_flag, + cancelled: false, + }; + let user_data = &mut ctx as *mut EpCtx as *mut std::ffi::c_void; + let callback: flEpProgressCallback = Some(ep_trampoline); + + // SAFETY: name pointers and `ctx` outlive this blocking call. + let status = unsafe { + (self.api.root().Manager_DownloadAndRegisterEps)( + self.ptr, names_ptr, names_len, callback, user_data, + ) + }; + self.api.status_message(status) + } + + /// Begin graceful shutdown of the native manager (`Manager_Shutdown`). + /// + /// Idempotent and safe to call from any thread. Does **not** release the + /// native handle — that happens once the last owner is dropped, via + /// [`teardown`](Self::teardown). + pub(crate) fn shutdown(&self) -> Result<()> { + if self.released.load(Ordering::Acquire) { + return Ok(()); + } + // SAFETY: `ptr` is a live manager handle (not yet released — guarded above). + let status = unsafe { (self.api.root().Manager_Shutdown)(self.ptr) }; + self.api.check(status) + } + + /// Run the prescribed teardown exactly once: `Manager_Shutdown` then + /// `Manager_Release`. + /// + /// Invoked from [`Drop`] when the last owner is released, so the manager's + /// C++ destructor runs *before* the library's static destructors — avoiding + /// the spdlog teardown abort (`mutex lock failed`) documented for the other + /// SDK bindings, and the WebGPU `ReleaseEpFactory` throw that a process-exit + /// release would trigger. Releasing is always attempted, even if shutdown + /// errored. + pub(crate) fn teardown(&self) { + if self.released.swap(true, Ordering::AcqRel) { + return; + } + // Serialize against a concurrent `create` so the native core never sees + // an overlapping create/release (see `NATIVE_LIFECYCLE`). + let _lifecycle = lock_lifecycle(); + // SAFETY: `ptr` was created by Manager_Create and is released exactly + // once (guarded by the `released` swap above). + unsafe { + let status = (self.api.root().Manager_Shutdown)(self.ptr); + if !status.is_null() { + (self.api.root().Status_Release)(status); + } + (self.api.root().Manager_Release)(self.ptr); + } + } +} + +impl Drop for NativeManager { + fn drop(&mut self) { + self.teardown(); + } +} + +/// Boxed `(ep_name, percent)` progress callback. +pub(crate) type EpProgressCallback = Box; + +struct EpCtx { + progress: Option, + cancel_flag: Option>, + cancelled: bool, +} + +unsafe extern "C" fn ep_trampoline( + ep_name: *const c_char, + value: f32, + user_data: *mut std::ffi::c_void, +) -> std::os::raw::c_int { + if user_data.is_null() { + return 0; + } + let result = catch_unwind(AssertUnwindSafe(|| { + let ctx = &mut *(user_data as *mut EpCtx); + if ctx + .cancel_flag + .as_ref() + .is_some_and(|f| f.load(Ordering::Relaxed)) + { + ctx.cancelled = true; + return 1; + } + if let Some(cb) = ctx.progress.as_mut() { + let name = cstr_to_string(ep_name).unwrap_or_default(); + cb(&name, value as f64); + } + 0 + })); + result.unwrap_or(1) +} diff --git a/sdk_v2/rust/src/detail/mod.rs b/sdk_v2/rust/src/detail/mod.rs new file mode 100644 index 000000000..f8e299ea4 --- /dev/null +++ b/sdk_v2/rust/src/detail/mod.rs @@ -0,0 +1,13 @@ +//! Internal implementation detail of the Foundry Local SDK. +//! +//! Nothing in this module is part of the public API. + +pub(crate) mod api; +pub(crate) mod ffi; +pub(crate) mod info; +pub(crate) mod items; +pub(crate) mod manager; +pub(crate) mod model; +pub(crate) mod native; +pub(crate) mod session; +pub(crate) mod task; diff --git a/sdk_v2/rust/src/detail/model.rs b/sdk_v2/rust/src/detail/model.rs new file mode 100644 index 000000000..537417c38 --- /dev/null +++ b/sdk_v2/rust/src/detail/model.rs @@ -0,0 +1,421 @@ +//! Public [`Model`] type backed by catalog-owned native handles. +//! +//! Mirrors the legacy SDK: a `Model` is either a single variant or a group of +//! variants sharing an alias. Selection is tracked Rust-side (an index) and all +//! operations delegate to the selected variant's native handle, so +//! [`Model::info`] / [`Model::id`] return references that always reflect the +//! current selection. + +use std::fmt; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::Relaxed}; +use std::sync::Arc; + +use super::api::Api; +use super::info::build_model_info; +use super::native::NativeModel; +use super::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; +use crate::types::ModelInfo; + +/// One specific variant: its native handle plus the cached, immutable metadata. +#[derive(Clone)] +pub(crate) struct VariantData { + native: NativeModel, + info: ModelInfo, +} + +/// The public model type. +/// +/// A `Model` may represent either a group of variants (as returned by +/// [`Catalog::get_model`](crate::Catalog::get_model)) or a single variant (as +/// returned by [`Catalog::get_model_variant`](crate::Catalog::get_model_variant) +/// or [`Model::variants`]). +pub struct Model { + inner: ModelKind, +} + +type DownloadProgressCallback = Box; + +/// Builder for configuring and running a model download. +/// +/// Use this builder when combining optional settings like progress and cancellation. +pub struct DownloadBuilder<'a> { + model: &'a Model, + progress: Option, + cancel_flag: Option>, +} + +impl<'a> DownloadBuilder<'a> { + fn new(model: &'a Model) -> Self { + Self { + model, + progress: None, + cancel_flag: None, + } + } + + /// Report download progress as a percentage from 0.0 to 100.0. + pub fn progress(mut self, callback: F) -> Self + where + F: FnMut(f64) + Send + 'static, + { + self.progress = Some(Box::new(callback)); + self + } + + /// Cancel the download when `cancel_flag` is set to `true`. + pub fn cancel(mut self, cancel_flag: Arc) -> Self { + self.cancel_flag = Some(cancel_flag); + self + } + + /// Run the configured download. + pub async fn run(self) -> Result<()> { + let native = self.model.selected_variant().native.clone(); + let progress = self.progress; + let cancel_flag = self.cancel_flag; + spawn_blocking(move || native.download(progress, cancel_flag)).await + } +} +enum ModelKind { + /// A single model variant (from `get_model_variant` or `variants()`). + Variant(Arc), + /// A group of variants sharing the same alias (from `get_model`). + Group { + alias: String, + variants: Vec>, + selected: AtomicUsize, + }, +} + +impl Clone for Model { + fn clone(&self) -> Self { + Self { + inner: match &self.inner { + ModelKind::Variant(v) => ModelKind::Variant(v.clone()), + ModelKind::Group { + alias, + variants, + selected, + } => ModelKind::Group { + alias: alias.clone(), + variants: variants.clone(), + selected: AtomicUsize::new(selected.load(Relaxed)), + }, + }, + } + } +} + +impl fmt::Debug for Model { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.inner { + ModelKind::Variant(v) => f + .debug_struct("Model::ModelVariant") + .field("id", &v.info.id) + .field("alias", &v.info.alias) + .finish(), + ModelKind::Group { + alias, + variants, + selected, + } => f + .debug_struct("Model::Model") + .field("alias", alias) + .field("id", &variants[selected.load(Relaxed)].info.id) + .field("variants_count", &variants.len()) + .field("selected_index", &selected.load(Relaxed)) + .finish(), + } + } +} + +// ── Construction (crate-internal) ──────────────────────────────────────────── + +impl Model { + /// Wrap a single leaf variant. + pub(crate) fn from_variant(api: &Arc, native: NativeModel) -> Result { + let info = build_model_info(api, &native)?; + Ok(Self { + inner: ModelKind::Variant(Arc::new(VariantData { native, info })), + }) + } + + /// Wrap an alias-group model, eagerly loading its variants. + pub(crate) fn from_group(api: &Arc, native: NativeModel) -> Result { + let group_info = build_model_info(api, &native)?; + let alias = group_info.alias.clone(); + + let mut variants = Vec::new(); + for variant_native in native.get_variants()? { + let info = build_model_info(api, &variant_native)?; + variants.push(Arc::new(VariantData { + native: variant_native, + info, + })); + } + + // A leaf masquerading as a group: fall back to a single-variant model. + if variants.is_empty() { + return Ok(Self { + inner: ModelKind::Variant(Arc::new(VariantData { + native, + info: group_info, + })), + }); + } + + // Prefer the first cached variant as the initial selection. + let selected = variants.iter().position(|v| v.info.cached).unwrap_or(0); + + Ok(Self { + inner: ModelKind::Group { + alias, + variants, + selected: AtomicUsize::new(selected), + }, + }) + } +} + +// ── Private helpers ────────────────────────────────────────────────────────── + +impl Model { + fn selected_variant(&self) -> &VariantData { + match &self.inner { + ModelKind::Variant(v) => v.as_ref(), + ModelKind::Group { + variants, selected, .. + } => variants[selected.load(Relaxed)].as_ref(), + } + } + + pub(crate) fn selected_native(&self) -> &NativeModel { + &self.selected_variant().native + } +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +impl Model { + /// Unique identifier of the (selected) variant. + pub fn id(&self) -> &str { + &self.selected_variant().info.id + } + + /// Alias shared by all variants of this model. + pub fn alias(&self) -> &str { + match &self.inner { + ModelKind::Variant(v) => &v.info.alias, + ModelKind::Group { alias, .. } => alias, + } + } + + /// Full catalog metadata for the (selected) variant. + pub fn info(&self) -> &ModelInfo { + &self.selected_variant().info + } + + /// Maximum context length (in tokens), or `None` if unknown. + pub fn context_length(&self) -> Option { + self.selected_variant().info.context_length + } + + /// Comma-separated input modalities (e.g. `"text,image"`), or `None`. + pub fn input_modalities(&self) -> Option<&str> { + self.selected_variant().info.input_modalities.as_deref() + } + + /// Comma-separated output modalities (e.g. `"text"`), or `None`. + pub fn output_modalities(&self) -> Option<&str> { + self.selected_variant().info.output_modalities.as_deref() + } + + /// Capability tags (e.g. `"reasoning"`), or `None`. + pub fn capabilities(&self) -> Option<&str> { + self.selected_variant().info.capabilities.as_deref() + } + + /// Whether the model supports tool/function calling, or `None`. + pub fn supports_tool_calling(&self) -> Option { + self.selected_variant().info.supports_tool_calling + } + + /// Whether the (selected) variant is cached on disk. + pub async fn is_cached(&self) -> Result { + let native = self.selected_native().clone(); + spawn_blocking(move || native.is_cached()).await + } + + /// Whether the (selected) variant is loaded into memory. + pub async fn is_loaded(&self) -> Result { + let native = self.selected_native().clone(); + spawn_blocking(move || native.is_loaded()).await + } + + /// Download the (selected) variant. If `progress` is provided it + /// receives download progress as a percentage (0.0–100.0). + pub async fn download(&self, progress: Option) -> Result<()> + where + F: FnMut(f64) + Send + 'static, + { + let native = self.selected_native().clone(); + let progress: Option = + progress.map(|f| Box::new(f) as DownloadProgressCallback); + spawn_blocking(move || native.download(progress, None)).await + } + + /// Configure and run a model download with a builder. + /// + /// Use this for call sites that need progress, cancellation, or future + /// download options. + pub fn download_builder(&self) -> DownloadBuilder<'_> { + DownloadBuilder::new(self) + } + + /// Return the local file-system path of the (selected) variant. + pub async fn path(&self) -> Result { + let native = self.selected_native().clone(); + let id = self.id().to_owned(); + let path = spawn_blocking(move || native.path()).await?; + match path { + Some(p) => Ok(PathBuf::from(p)), + None => Err(FoundryLocalError::ModelOperation { + reason: format!("Error getting path for model {id}. Has it been downloaded?"), + }), + } + } + + /// Load the (selected) variant into memory. + pub async fn load(&self) -> Result<()> { + let native = self.selected_native().clone(); + spawn_blocking(move || native.load()).await + } + + /// Unload the (selected) variant from memory. + pub async fn unload(&self) -> Result<()> { + let native = self.selected_native().clone(); + spawn_blocking(move || native.unload()).await + } + + /// Remove the (selected) variant from the local cache. + pub async fn remove_from_cache(&self) -> Result<()> { + let native = self.selected_native().clone(); + spawn_blocking(move || native.remove_from_cache()).await + } + + /// Create a [`ChatClient`](crate::openai::ChatClient) bound to the (selected) variant. + #[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use `ChatSession::new(&model)` instead." + )] + #[allow(deprecated)] + pub fn create_chat_client(&self) -> crate::openai::ChatClient { + let v = self.selected_variant(); + crate::openai::ChatClient::new(&v.info.id, v.native.clone()) + } + + /// Create an [`AudioClient`](crate::openai::AudioClient) bound to the (selected) variant. + #[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use `AudioSession::new(&model)` instead." + )] + #[allow(deprecated)] + pub fn create_audio_client(&self) -> crate::openai::AudioClient { + let v = self.selected_variant(); + crate::openai::AudioClient::new(&v.info.id, v.native.clone()) + } + + /// Create an [`EmbeddingClient`](crate::openai::EmbeddingClient) bound to the (selected) variant. + #[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use `EmbeddingsSession::new(&model)` \ + instead." + )] + #[allow(deprecated)] + pub fn create_embedding_client(&self) -> crate::openai::EmbeddingClient { + let v = self.selected_variant(); + crate::openai::EmbeddingClient::new(&v.info.id, v.native.clone()) + } + + /// Available variants of this model. + /// + /// For a single-variant model (e.g. from + /// [`Catalog::get_model_variant`](crate::Catalog::get_model_variant)), + /// this returns a single-element list containing itself. + pub fn variants(&self) -> Vec> { + match &self.inner { + ModelKind::Variant(v) => { + vec![Arc::new(Model { + inner: ModelKind::Variant(v.clone()), + })] + } + ModelKind::Group { variants, .. } => variants + .iter() + .map(|v| { + Arc::new(Model { + inner: ModelKind::Variant(v.clone()), + }) + }) + .collect(), + } + } + + /// Select a variant to use for subsequent operations. + /// + /// The `variant` must be one of the models returned by [`variants`](Model::variants). + /// + /// # Errors + /// + /// Returns an error if the variant does not belong to this model. + /// For single-variant models this always returns an error — use + /// [`Catalog::get_model`](crate::Catalog::get_model) to obtain a model + /// with all variants available. + pub fn select_variant(&self, variant: &Model) -> Result<()> { + self.select_variant_by_id(variant.id()) + } + + /// Select a variant by its unique id string. + /// + /// This is a convenience method for cases where you have a variant id + /// from an external source. Prefer [`select_variant`](Model::select_variant) + /// when you already have a [`Model`] reference from [`variants`](Model::variants). + /// + /// # Errors + /// + /// Returns an error if no variant with the given id exists. + /// For single-variant models this always returns an error — use + /// [`Catalog::get_model`](crate::Catalog::get_model) to obtain a model + /// with all variants available. + pub fn select_variant_by_id(&self, id: &str) -> Result<()> { + match &self.inner { + ModelKind::Variant(v) => Err(FoundryLocalError::ModelOperation { + reason: format!( + "Selecting a variant is not supported on a single-variant model. \ + Call Catalog::get_model(\"{}\") to get a model with all variants available.", + v.info.alias + ), + }), + ModelKind::Group { + variants, + selected, + alias, + } => match variants.iter().position(|v| v.info.id == id) { + Some(pos) => { + selected.store(pos, Relaxed); + Ok(()) + } + None => { + let available: Vec<&str> = + variants.iter().map(|v| v.info.id.as_str()).collect(); + Err(FoundryLocalError::ModelOperation { + reason: format!( + "Variant '{id}' not found for model '{alias}'. Available: {available:?}", + ), + }) + } + }, + } + } +} diff --git a/sdk_v2/rust/src/detail/native.rs b/sdk_v2/rust/src/detail/native.rs new file mode 100644 index 000000000..f13eda343 --- /dev/null +++ b/sdk_v2/rust/src/detail/native.rs @@ -0,0 +1,282 @@ +//! Wrappers around native `flModel` / `flCatalog` handles. +//! +//! `flModel*` and `flCatalog*` handles are owned by the native manager and are +//! never released individually; they stay valid only while that manager is +//! alive. Each wrapper therefore holds a strong [`Arc`] so a +//! handle can safely outlive the +//! [`FoundryLocalManager`](crate::FoundryLocalManager) that created it. A +//! `flModelList*`, by contrast, is owned by the caller: [`collect_models`] +//! eagerly extracts the contained handles and then releases the list. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use super::api::{cstr_to_string, Api}; +use super::ffi::*; +use super::manager::NativeManager; +use crate::error::{FoundryLocalError, Result}; + +/// A handle to a manager-owned `flModel`, plus a strong reference to the owning +/// native manager so the catalog (and therefore this handle) stays alive for as +/// long as the model is held. +#[derive(Clone)] +pub(crate) struct NativeModel { + pub(crate) api: Arc, + pub(crate) ptr: *mut flModel, + /// Keeps the native manager — which owns the catalog and this `flModel` — + /// alive. Propagated to derived models/sessions; never dereferenced here. + manager: Arc, +} + +// SAFETY: the `flModel` is owned by the native manager kept alive via `manager`, +// and the native implementation is thread-safe for independent operations. +unsafe impl Send for NativeModel {} +unsafe impl Sync for NativeModel {} + +impl NativeModel { + pub(crate) fn new(api: Arc, ptr: *mut flModel, manager: Arc) -> Self { + Self { api, ptr, manager } + } + + /// Clone the keep-alive handle to the owning native manager. + pub(crate) fn manager(&self) -> Arc { + Arc::clone(&self.manager) + } + + pub(crate) fn info_ptr(&self) -> Result<*const flModelInfo> { + let mut info: *const flModelInfo = std::ptr::null(); + // SAFETY: `ptr` is a valid catalog-owned model handle. + let status = unsafe { (self.api.model_api().GetInfo)(self.ptr, &mut info) }; + self.api.check(status)?; + Ok(info) + } + + pub(crate) fn is_cached(&self) -> Result { + let mut cached: std::os::raw::c_int = 0; + let status = unsafe { (self.api.model_api().IsCached)(self.ptr, &mut cached) }; + self.api.check(status)?; + Ok(cached != 0) + } + + pub(crate) fn is_loaded(&self) -> Result { + let mut loaded: std::os::raw::c_int = 0; + let status = unsafe { (self.api.model_api().IsLoaded)(self.ptr, &mut loaded) }; + self.api.check(status)?; + Ok(loaded != 0) + } + + pub(crate) fn path(&self) -> Result> { + let mut path: *const std::os::raw::c_char = std::ptr::null(); + let status = unsafe { (self.api.model_api().GetPath)(self.ptr, &mut path) }; + self.api.check(status)?; + // SAFETY: `path`, when non-null, points to model-owned storage valid now. + Ok(unsafe { cstr_to_string(path) }) + } + + pub(crate) fn load(&self) -> Result<()> { + let status = unsafe { (self.api.model_api().Load)(self.ptr) }; + self.api.check(status) + } + + pub(crate) fn unload(&self) -> Result<()> { + let status = unsafe { (self.api.model_api().Unload)(self.ptr) }; + self.api.check(status) + } + + pub(crate) fn remove_from_cache(&self) -> Result<()> { + let status = unsafe { (self.api.model_api().RemoveFromCache)(self.ptr) }; + self.api.check(status) + } + + pub(crate) fn get_variants(&self) -> Result> { + let mut list: *mut flModelList = std::ptr::null_mut(); + let status = unsafe { (self.api.model_api().GetVariants)(self.ptr, &mut list) }; + self.api.check(status)?; + Ok(collect_models(&self.api, &self.manager, list)) + } + + /// Download the model, optionally reporting progress (0.0–100.0) and + /// honouring a cancellation flag. Blocking. + pub(crate) fn download( + &self, + progress: Option>, + cancel_flag: Option>, + ) -> Result<()> { + let mut ctx = DownloadCtx { + progress, + cancel_flag, + cancelled: false, + }; + let callback: flProgressCallback = Some(download_trampoline); + let user_data = &mut ctx as *mut DownloadCtx as *mut std::ffi::c_void; + // SAFETY: `ctx` outlives the blocking native call; the trampoline only + // dereferences `user_data` for the duration of the call. + let status = unsafe { (self.api.model_api().Download)(self.ptr, callback, user_data) }; + if ctx.cancelled { + self.api.check(status).ok(); + return Err(FoundryLocalError::CommandExecution { + reason: "Operation cancelled".into(), + }); + } + self.api.check(status) + } +} + +struct DownloadCtx { + progress: Option>, + cancel_flag: Option>, + cancelled: bool, +} + +unsafe extern "C" fn download_trampoline( + value: f32, + user_data: *mut std::ffi::c_void, +) -> std::os::raw::c_int { + if user_data.is_null() { + return 0; + } + let result = catch_unwind(AssertUnwindSafe(|| { + let ctx = &mut *(user_data as *mut DownloadCtx); + if ctx + .cancel_flag + .as_ref() + .is_some_and(|f| f.load(Ordering::Relaxed)) + { + ctx.cancelled = true; + return 1; + } + if let Some(cb) = ctx.progress.as_mut() { + cb(value as f64); + } + 0 + })); + result.unwrap_or(1) +} + +/// Eagerly extract all model handles from a `flModelList`, then release the list. +/// +/// Each extracted handle is tagged with `manager` so it keeps the owning native +/// manager alive for as long as the handle is held. +pub(crate) fn collect_models( + api: &Arc, + manager: &Arc, + list: *mut flModelList, +) -> Vec { + if list.is_null() { + return Vec::new(); + } + let root = api.root(); + // SAFETY: `list` is a valid list handle we own until released below. + let size = unsafe { (root.ModelList_Size)(list) }; + let mut out = Vec::with_capacity(size); + for i in 0..size { + let m = unsafe { (root.ModelList_GetAt)(list, i) }; + if !m.is_null() { + out.push(NativeModel::new(Arc::clone(api), m, Arc::clone(manager))); + } + } + unsafe { (root.ModelList_Release)(list) }; + out +} + +/// A handle to a manager-owned `flCatalog`, plus a strong reference to the +/// owning native manager so the handle (and the models it produces) outlive the +/// [`FoundryLocalManager`](crate::FoundryLocalManager). +#[derive(Clone)] +pub(crate) struct NativeCatalog { + pub(crate) api: Arc, + pub(crate) ptr: *mut flCatalog, + /// Keeps the native manager that owns this `flCatalog` alive; also tagged + /// onto every model produced from this catalog. + manager: Arc, +} + +// SAFETY: the `flCatalog` is owned by the native manager kept alive via +// `manager`, and the native implementation is thread-safe. +unsafe impl Send for NativeCatalog {} +unsafe impl Sync for NativeCatalog {} + +impl NativeCatalog { + pub(crate) fn new(api: Arc, ptr: *mut flCatalog, manager: Arc) -> Self { + Self { api, ptr, manager } + } + + pub(crate) fn name(&self) -> Result { + let mut name: *const std::os::raw::c_char = std::ptr::null(); + let status = unsafe { (self.api.catalog_api().GetName)(self.ptr, &mut name) }; + self.api.check(status)?; + Ok(unsafe { cstr_to_string(name) }.unwrap_or_default()) + } + + fn list( + &self, + f: unsafe extern "system" fn(*const flCatalog, *mut *mut flModelList) -> flStatusPtr, + ) -> Result> { + let mut list: *mut flModelList = std::ptr::null_mut(); + let status = unsafe { f(self.ptr, &mut list) }; + self.api.check(status)?; + Ok(collect_models(&self.api, &self.manager, list)) + } + + pub(crate) fn get_models(&self) -> Result> { + self.list(self.api.catalog_api().GetModels) + } + + pub(crate) fn get_cached_models(&self) -> Result> { + self.list(self.api.catalog_api().GetCachedModels) + } + + pub(crate) fn get_loaded_models(&self) -> Result> { + self.list(self.api.catalog_api().GetLoadedModels) + } + + fn lookup( + &self, + f: unsafe extern "system" fn( + *const flCatalog, + *const std::os::raw::c_char, + *mut *mut flModel, + ) -> flStatusPtr, + key: &str, + ) -> Result> { + let c = super::api::to_cstring(key)?; + let mut model: *mut flModel = std::ptr::null_mut(); + let status = unsafe { f(self.ptr, c.as_ptr(), &mut model) }; + self.api.check(status)?; + if model.is_null() { + Ok(None) + } else { + Ok(Some(NativeModel::new( + Arc::clone(&self.api), + model, + Arc::clone(&self.manager), + ))) + } + } + + pub(crate) fn get_model(&self, alias: &str) -> Result> { + self.lookup(self.api.catalog_api().GetModel, alias) + } + + pub(crate) fn get_model_variant(&self, model_id: &str) -> Result> { + self.lookup(self.api.catalog_api().GetModelVariant, model_id) + } + + pub(crate) fn get_latest_version(&self, model: &NativeModel) -> Result { + let mut latest: *mut flModel = std::ptr::null_mut(); + let status = + unsafe { (self.api.catalog_api().GetLatestVersion)(self.ptr, model.ptr, &mut latest) }; + self.api.check(status)?; + if latest.is_null() { + return Err(FoundryLocalError::ModelOperation { + reason: "Catalog returned no latest version for the model.".into(), + }); + } + Ok(NativeModel::new( + Arc::clone(&self.api), + latest, + Arc::clone(&self.manager), + )) + } +} diff --git a/sdk_v2/rust/src/detail/session.rs b/sdk_v2/rust/src/detail/session.rs new file mode 100644 index 000000000..d4a99333f --- /dev/null +++ b/sdk_v2/rust/src/detail/session.rs @@ -0,0 +1,618 @@ +//! Safe wrappers over native `flRequest`, `flResponse`, and `flSession`, plus the +//! OpenAI-JSON request/response and streaming bridges used by the OpenAI facade. + +use std::os::raw::c_int; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::ptr; +use std::sync::Arc; + +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; + +use super::api::{Api, Kvps}; +use super::ffi::*; +use super::items::{ + item_from_native, item_to_native, make_bytes_item, make_openai_json_item, + read_speech_result_text, read_text_item, +}; +use super::manager::NativeManager; +use super::native::NativeModel; +use crate::error::{FoundryLocalError, Result}; +use crate::item::Item; + +/// Per-item transform applied to streamed TEXT payloads before they are emitted. +pub(crate) type StreamTransform = Box Option + Send>; + +// ── Request ────────────────────────────────────────────────────────────────── + +pub(crate) struct NativeRequest { + api: Arc, + ptr: *mut flRequest, +} + +// SAFETY: the raw `flRequest` pointer is only *mutated* (add_item / set_options / +// process) from the single blocking worker that owns the request. The one method +// callable from another thread is `cancel`, which the native layer implements as +// an atomic-flag store (`Request_Cancel`), so concurrent cancel-vs-process is +// well-defined. This mirrors the Send+Sync story of `NativeSession`. +unsafe impl Send for NativeRequest {} +unsafe impl Sync for NativeRequest {} + +impl NativeRequest { + pub(crate) fn new(api: Arc) -> Result { + let mut ptr: *mut flRequest = ptr::null_mut(); + api.check(unsafe { (api.inference_api().Request_Create)(&mut ptr) })?; + Ok(Self { api, ptr }) + } + + /// Add an item, transferring ownership to the request. + pub(crate) fn add_item(&self, item: *mut flItem, take_ownership: bool) -> Result<()> { + let status = + unsafe { (self.api.inference_api().Request_AddItem)(self.ptr, item, take_ownership) }; + self.api.check(status) + } + + /// Build a native item from a public [`Item`] and add it, transferring + /// ownership to the request. Releases the transient item if the add fails. + pub(crate) fn add_item_value(&self, item: &Item) -> Result<()> { + let native = item_to_native(&self.api, item)?; + if let Err(e) = self.add_item(native, true) { + // The add did not take ownership on failure — reclaim to avoid a leak. + unsafe { (self.api.item_api().Item_Release)(native) }; + return Err(e); + } + Ok(()) + } + + /// Add a streaming input queue to the request as a *borrowed* item: the queue + /// remains owned by its [`NativeItemQueue`] and must outlive processing. + pub(crate) fn add_input_queue(&self, queue: &NativeItemQueue) -> Result<()> { + self.add_item(queue.as_item_ptr(), false) + } + + /// Apply request-scoped options from a native key/value collection. + pub(crate) fn set_options(&self, options: *const flKeyValuePairs) -> Result<()> { + let status = unsafe { (self.api.inference_api().Request_SetOptions)(self.ptr, options) }; + self.api.check(status) + } + + /// Signal cancellation of an in-flight request. + /// + /// The native layer records this as an atomic flag (`Request_Cancel`), so it + /// is safe to call from a thread other than the one running + /// [`NativeSession::process_request`]; in-progress generation stops as soon + /// as possible. Best-effort: any status is released and the error ignored, + /// which is appropriate for the drop-cancel path. + pub(crate) fn cancel(&self) { + let status = unsafe { (self.api.inference_api().Request_Cancel)(self.ptr) }; + let _ = self.api.check(status); + } +} + +impl Drop for NativeRequest { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { (self.api.inference_api().Request_Release)(self.ptr) }; + self.ptr = ptr::null_mut(); + } + } +} + +// ── Response ───────────────────────────────────────────────────────────────── + +pub(crate) struct NativeResponse { + api: Arc, + ptr: *mut flResponse, +} + +impl NativeResponse { + pub(crate) fn item_count(&self) -> usize { + unsafe { (self.api.inference_api().Response_GetItemCount)(self.ptr) } + } + + /// Read the text payload of the response item at `idx` (if it is a TEXT item). + pub(crate) fn item_text(&self, idx: usize) -> Option { + let mut item: *const flItem = ptr::null(); + let status = + unsafe { (self.api.inference_api().Response_GetItem)(self.ptr, idx, &mut item) }; + if self.api.check(status).is_err() { + return None; + } + unsafe { read_text_item(&self.api, item) } + } + + /// Read the transcript of the response item at `idx` (if it is a SPEECH_RESULT item). + pub(crate) fn item_speech_result_text(&self, idx: usize) -> Option { + let mut item: *const flItem = ptr::null(); + let status = + unsafe { (self.api.inference_api().Response_GetItem)(self.ptr, idx, &mut item) }; + if self.api.check(status).is_err() { + return None; + } + unsafe { read_speech_result_text(&self.api, item) } + } + + /// Read the response item at `idx` into an owned [`Item`]. + pub(crate) fn item(&self, idx: usize) -> Option { + let mut item: *const flItem = ptr::null(); + let status = + unsafe { (self.api.inference_api().Response_GetItem)(self.ptr, idx, &mut item) }; + if self.api.check(status).is_err() { + return None; + } + unsafe { item_from_native(&self.api, item) } + } + + /// Collect all response items into owned [`Item`]s. + pub(crate) fn items(&self) -> Vec { + let count = self.item_count(); + let mut out = Vec::with_capacity(count); + for i in 0..count { + if let Some(item) = self.item(i) { + out.push(item); + } + } + out + } + + /// The native finish-reason discriminant for the response. + pub(crate) fn finish_reason(&self) -> flFinishReason { + unsafe { (self.api.inference_api().Response_GetFinishReason)(self.ptr) } + } + + /// Token usage as `(prompt, completion, total)`; zeros if unavailable. + pub(crate) fn usage(&self) -> (i64, i64, i64) { + let mut usage = flUsage { + version: FOUNDRY_LOCAL_API_VERSION, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }; + let status = unsafe { (self.api.inference_api().Response_GetUsage)(self.ptr, &mut usage) }; + if self.api.check(status).is_err() { + return (0, 0, 0); + } + ( + usage.prompt_tokens, + usage.completion_tokens, + usage.total_tokens, + ) + } +} + +impl Drop for NativeResponse { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { (self.api.inference_api().Response_Release)(self.ptr) }; + self.ptr = ptr::null_mut(); + } + } +} + +// ── ItemQueue ──────────────────────────────────────────────────────────────── + +/// Owning wrapper around a native input `flItemQueue`. +/// +/// In the C ABI an `ItemQueue` *is* an `Item` (same pointer, castable), so it can +/// be added to a request directly and released via `Item_Release`. +pub(crate) struct NativeItemQueue { + api: Arc, + ptr: *mut flItemQueue, +} + +// SAFETY: the native item queue is documented as thread-safe (multi-producer / +// multi-consumer); pushing from any thread is supported. +unsafe impl Send for NativeItemQueue {} +unsafe impl Sync for NativeItemQueue {} + +impl NativeItemQueue { + pub(crate) fn new(api: Arc) -> Result { + let mut ptr: *mut flItemQueue = ptr::null_mut(); + api.check(unsafe { (api.item_api().ItemQueue_Create)(&mut ptr) })?; + Ok(Self { api, ptr }) + } + + /// The queue as an `flItem*` (for adding to a request). + pub(crate) fn as_item_ptr(&self) -> *mut flItem { + self.ptr as *mut flItem + } + + /// Push an item, transferring ownership into the queue. + /// + /// `ItemQueue_Push` takes ownership of `item` *unconditionally* for a + /// non-null queue and item: the native side moves the raw pointer into a + /// `unique_ptr` before enqueuing, so even if enqueuing fails the item is + /// already (or will be) freed. Callers must therefore **not** release `item` + /// on a returned error — doing so would double-free. + pub(crate) fn push_item(&self, item: *mut flItem) -> Result<()> { + self.api + .check(unsafe { (self.api.item_api().ItemQueue_Push)(self.ptr, item) }) + } + + /// Create a BYTES item from `data` and push it into the queue. + pub(crate) fn push_bytes(&self, data: &[u8], item_type: flItemType) -> Result<()> { + let item = make_bytes_item(&self.api, data, item_type)?; + // `push_item` consumes `item` on every path (see its docs); do not + // release it here on error. + self.push_item(item) + } + + /// Build a native item from a public [`Item`] and push it into the queue. + pub(crate) fn push_value(&self, item: &Item) -> Result<()> { + let native = item_to_native(&self.api, item)?; + // `push_item` consumes `native` on every path (see its docs). + self.push_item(native) + } + + /// Pop the next item, if any, decoding it into an owned [`Item`]. + /// + /// Returns `None` when the queue is currently empty. Ownership of the popped + /// native item transfers to us, so it is released after decoding. + pub(crate) fn try_pop_value(&self) -> Option { + let mut item: *mut flItem = ptr::null_mut(); + let popped = unsafe { (self.api.item_api().ItemQueue_TryPop)(self.ptr, &mut item) }; + if !popped || item.is_null() { + return None; + } + let decoded = unsafe { item_from_native(&self.api, item) }; + unsafe { (self.api.item_api().Item_Release)(item) }; + decoded + } + + /// The number of items currently buffered in the queue. + pub(crate) fn size(&self) -> usize { + unsafe { (self.api.item_api().ItemQueue_Size)(self.ptr) } + } + + /// Whether the queue has been marked finished (no further pushes expected). + pub(crate) fn is_finished(&self) -> bool { + unsafe { (self.api.item_api().ItemQueue_IsFinished)(self.ptr) } + } + + /// Signal that no more items will be pushed. + pub(crate) fn mark_finished(&self) { + unsafe { (self.api.item_api().ItemQueue_MarkFinished)(self.ptr) }; + } +} + +impl Drop for NativeItemQueue { + fn drop(&mut self) { + if !self.ptr.is_null() { + // The queue is an Item; release via the polymorphic Item destructor. + unsafe { (self.api.item_api().Item_Release)(self.ptr as *mut flItem) }; + self.ptr = ptr::null_mut(); + } + } +} + +// ── Session ────────────────────────────────────────────────────────────────── + +pub(crate) struct NativeSession { + pub(crate) api: Arc, + ptr: *mut flSession, + /// Keeps the native manager (which owns the model this session was created + /// from) alive for the session's lifetime; never dereferenced here. + _manager: Arc, +} + +// SAFETY: a session is used from a single worker at a time; the native layer is +// thread-safe for the create/process/release lifecycle used here. The owning +// native manager is kept alive via `_manager`. +unsafe impl Send for NativeSession {} +unsafe impl Sync for NativeSession {} + +impl NativeSession { + /// Create a session bound to the given model variant. + pub(crate) fn create(model: &NativeModel) -> Result { + let api = Arc::clone(&model.api); + let mut ptr: *mut flSession = ptr::null_mut(); + api.check(unsafe { (api.inference_api().Session_Create)(model.ptr, &mut ptr) })?; + Ok(Self { + api, + ptr, + _manager: model.manager(), + }) + } + + pub(crate) fn set_streaming_callback( + &self, + callback: flStreamingCallback, + user_data: *mut std::ffi::c_void, + ) -> Result<()> { + let status = unsafe { + (self.api.inference_api().Session_SetStreamingCallback)(self.ptr, callback, user_data) + }; + self.api.check(status) + } + + /// Apply session-scoped options from a native key/value collection. + pub(crate) fn set_options(&self, options: *const flKeyValuePairs) -> Result<()> { + let status = unsafe { (self.api.inference_api().Session_SetOptions)(self.ptr, options) }; + self.api.check(status) + } + + /// Register a tool definition for the lifetime of the session. + pub(crate) fn add_tool_definition( + &self, + name: &str, + description: Option<&str>, + json_schema: &str, + ) -> Result<()> { + let name_c = super::api::to_cstring(name)?; + let desc_c = match description { + Some(d) => Some(super::api::to_cstring(d)?), + None => None, + }; + let schema_c = super::api::to_cstring(json_schema)?; + let def = flToolDefinition { + version: FOUNDRY_LOCAL_API_VERSION, + name: name_c.as_ptr(), + description: desc_c.as_ref().map_or(ptr::null(), |c| c.as_ptr()), + json_schema: schema_c.as_ptr(), + }; + let status = + unsafe { (self.api.inference_api().Session_AddToolDefinition)(self.ptr, &def) }; + self.api.check(status) + } + + /// Remove a previously-registered tool by name. Returns whether one was removed. + pub(crate) fn remove_tool_definition(&self, name: &str) -> Result { + let name_c = super::api::to_cstring(name)?; + let mut removed = false; + let status = unsafe { + (self.api.inference_api().Session_RemoveToolDefinition)( + self.ptr, + name_c.as_ptr(), + &mut removed, + ) + }; + self.api.check(status)?; + Ok(removed) + } + + /// The number of completed turns. + pub(crate) fn turn_count(&self) -> usize { + unsafe { (self.api.inference_api().Session_GetTurnCount)(self.ptr) } + } + + /// Rewind the last `count` turns, dropping their messages and replies. + pub(crate) fn undo_turns(&self, count: usize) -> Result<()> { + let status = unsafe { (self.api.inference_api().Session_UndoTurns)(self.ptr, count) }; + self.api.check(status) + } + + pub(crate) fn process_request(&self, request: &NativeRequest) -> Result { + let mut resp: *mut flResponse = ptr::null_mut(); + let status = unsafe { + (self.api.inference_api().Session_ProcessRequest)(self.ptr, request.ptr, &mut resp) + }; + self.api.check(status)?; + Ok(NativeResponse { + api: Arc::clone(&self.api), + ptr: resp, + }) + } + + /// Run a non-streaming OpenAI-JSON request and return the response payload. + /// + /// The request JSON is sent as a single `OPENAI_JSON` TEXT item; the response + /// payload is the text of the first response item. Blocking. + pub(crate) fn run_openai_json(&self, request_json: &str) -> Result { + let request = NativeRequest::new(Arc::clone(&self.api))?; + let item = make_openai_json_item(&self.api, request_json)?; + request.add_item(item, true)?; + let response = self.process_request(&request)?; + if response.item_count() == 0 { + return Err(FoundryLocalError::CommandExecution { + reason: "Native response contained no items".into(), + }); + } + response + .item_text(0) + .ok_or_else(|| FoundryLocalError::CommandExecution { + reason: "Native response item was not readable text".into(), + }) + } +} + +impl Drop for NativeSession { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { (self.api.inference_api().Session_Release)(self.ptr) }; + self.ptr = ptr::null_mut(); + } + } +} + +// ── Streaming bridge ───────────────────────────────────────────────────────── + +struct StreamCtx { + api: Arc, + tx: UnboundedSender>, + transform: StreamTransform, +} + +unsafe extern "C" fn stream_trampoline( + data: flStreamingCallbackData, + user_data: *mut std::ffi::c_void, +) -> c_int { + if user_data.is_null() { + return 0; + } + let result = catch_unwind(AssertUnwindSafe(|| { + let ctx = &*(user_data as *const StreamCtx); + let queue = data.item_queue; + if queue.is_null() { + return 0; + } + let item_api = ctx.api.item_api(); + loop { + let mut item: *mut flItem = ptr::null_mut(); + let popped = (item_api.ItemQueue_TryPop)(queue, &mut item); + if !popped { + break; + } + if item.is_null() { + continue; + } + // Ownership of `item` transferred to us — read then release. + let text = read_text_item(&ctx.api, item); + (item_api.Item_Release)(item); + if let Some(text) = text { + if let Some(transformed) = (ctx.transform)(text) { + if ctx.tx.send(Ok(transformed)).is_err() { + return 1; // receiver dropped — cancel generation + } + } + } + } + 0 + })); + result.unwrap_or(1) +} + +/// Run a streaming OpenAI-JSON request, returning a channel of transformed +/// per-item TEXT payloads. +/// +/// `transform` is applied to each streamed item's text (return `None` to skip +/// an item). The session is created and processed on a blocking worker thread; +/// the channel closes when generation completes or errors. +pub(crate) fn run_openai_json_streaming( + session: NativeSession, + request_json: String, + transform: StreamTransform, +) -> UnboundedReceiver> { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + + tokio::task::spawn_blocking(move || { + let ctx = Box::new(StreamCtx { + api: Arc::clone(&session.api), + tx: tx.clone(), + transform, + }); + let ctx_ptr = &*ctx as *const StreamCtx as *mut std::ffi::c_void; + + if let Err(e) = session.set_streaming_callback(Some(stream_trampoline), ctx_ptr) { + let _ = tx.send(Err(e)); + return; + } + + let run = (|| -> Result<()> { + let request = NativeRequest::new(Arc::clone(&session.api))?; + let item = make_openai_json_item(&session.api, &request_json)?; + request.add_item(item, true)?; + let _response = session.process_request(&request)?; + Ok(()) + })(); + if let Err(e) = run { + let _ = tx.send(Err(e)); + } + + // Uninstall the callback before the context/session are dropped. + let _ = session.set_streaming_callback(None, ptr::null_mut()); + drop(ctx); + drop(session); + }); + + rx +} + +// ── Generic item streaming bridge ───────────────────────────────────────────── + +struct ItemStreamCtx { + api: Arc, + tx: UnboundedSender>, +} + +unsafe extern "C" fn item_stream_trampoline( + data: flStreamingCallbackData, + user_data: *mut std::ffi::c_void, +) -> c_int { + if user_data.is_null() { + return 0; + } + let result = catch_unwind(AssertUnwindSafe(|| { + let ctx = &*(user_data as *const ItemStreamCtx); + let queue = data.item_queue; + if queue.is_null() { + return 0; + } + let item_api = ctx.api.item_api(); + loop { + let mut item: *mut flItem = ptr::null_mut(); + let popped = (item_api.ItemQueue_TryPop)(queue, &mut item); + if !popped { + break; + } + if item.is_null() { + continue; + } + // Ownership of `item` transferred to us — decode then release. + let decoded = item_from_native(&ctx.api, item); + (item_api.Item_Release)(item); + if let Some(decoded) = decoded { + if ctx.tx.send(Ok(decoded)).is_err() { + return 1; // receiver dropped — cancel generation + } + } + } + 0 + })); + result.unwrap_or(1) +} + +/// Run a request on a blocking worker, streaming each output item back through a +/// channel. +/// +/// Input is provided as owned `items` plus an optional borrowed `input_queue`; +/// `option_pairs` are applied as request options. The channel closes when +/// generation completes or errors; dropping the receiver cancels generation. +pub(crate) fn run_item_streaming( + session: Arc, + items: Vec, + input_queue: Option>, + option_pairs: Vec<(String, String)>, +) -> UnboundedReceiver> { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + + tokio::task::spawn_blocking(move || { + let ctx = Box::new(ItemStreamCtx { + api: Arc::clone(&session.api), + tx: tx.clone(), + }); + let ctx_ptr = &*ctx as *const ItemStreamCtx as *mut std::ffi::c_void; + + if let Err(e) = session.set_streaming_callback(Some(item_stream_trampoline), ctx_ptr) { + let _ = tx.send(Err(e)); + return; + } + + let run = (|| -> Result<()> { + let request = NativeRequest::new(Arc::clone(&session.api))?; + for item in &items { + request.add_item_value(item)?; + } + if let Some(queue) = &input_queue { + request.add_input_queue(queue)?; + } + if !option_pairs.is_empty() { + let kvps = Kvps::from_pairs( + Arc::clone(&session.api), + option_pairs.iter().map(|(k, v)| (k.as_str(), v.as_str())), + )?; + request.set_options(kvps.as_ptr())?; + } + let _response = session.process_request(&request)?; + Ok(()) + })(); + if let Err(e) = run { + let _ = tx.send(Err(e)); + } + + // Uninstall the callback before the context/session are dropped. + let _ = session.set_streaming_callback(None, ptr::null_mut()); + drop(ctx); + drop(session); + }); + + rx +} diff --git a/sdk_v2/rust/src/detail/task.rs b/sdk_v2/rust/src/detail/task.rs new file mode 100644 index 000000000..36b5679e6 --- /dev/null +++ b/sdk_v2/rust/src/detail/task.rs @@ -0,0 +1,16 @@ +//! Shared async helper for running blocking native calls. + +use crate::error::{FoundryLocalError, Result}; + +/// Run a blocking native operation on the tokio blocking pool. +pub(crate) async fn spawn_blocking(f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(f) + .await + .map_err(|e| FoundryLocalError::Internal { + reason: format!("blocking task join error: {e}"), + })? +} diff --git a/sdk_v2/rust/src/error.rs b/sdk_v2/rust/src/error.rs new file mode 100644 index 000000000..c99dbfbf1 --- /dev/null +++ b/sdk_v2/rust/src/error.rs @@ -0,0 +1,36 @@ +use thiserror::Error; + +/// Errors that can occur when using the Foundry Local SDK. +#[derive(Debug, Error)] +pub enum FoundryLocalError { + /// The native core library could not be loaded. + #[error("library load error: {reason}")] + LibraryLoad { reason: String }, + /// A command executed against the native core returned an error. + #[error("command execution error: {reason}")] + CommandExecution { reason: String }, + /// The provided configuration is invalid. + #[error("invalid configuration: {reason}")] + InvalidConfiguration { reason: String }, + /// A model operation failed (load, unload, download, etc.). + #[error("model operation error: {reason}")] + ModelOperation { reason: String }, + /// An HTTP request to the external service failed. + #[error("HTTP request error: {0}")] + HttpRequest(#[from] reqwest::Error), + /// Serialization or deserialization of JSON data failed. + #[error("serialization error: {0}")] + Serialization(#[from] serde_json::Error), + /// A validation check on user-supplied input failed. + #[error("validation error: {reason}")] + Validation { reason: String }, + /// An I/O error occurred. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + /// An internal SDK error (e.g. poisoned lock). + #[error("internal error: {reason}")] + Internal { reason: String }, +} + +/// Convenience alias used throughout the SDK. +pub type Result = std::result::Result; diff --git a/sdk_v2/rust/src/foundry_local_manager.rs b/sdk_v2/rust/src/foundry_local_manager.rs new file mode 100644 index 000000000..095b834a9 --- /dev/null +++ b/sdk_v2/rust/src/foundry_local_manager.rs @@ -0,0 +1,327 @@ +//! Top-level entry point for the Foundry Local SDK. +//! +//! [`FoundryLocalManager`] initialises the native core library, provides access +//! to the model [`Catalog`], and can start / stop the local web service. While a +//! handle is alive it is shared process-wide (see [`FoundryLocalManager::create`]). + +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex, OnceLock, Weak}; + +use crate::catalog::Catalog; +use crate::configuration::{FoundryLocalConfig, Logger}; +use crate::detail::api::Api; +use crate::detail::manager::{EpProgressCallback, NativeManager}; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; +use crate::types::{EpDownloadResult, EpInfo}; + +/// Process-wide weak handle to the live manager. +/// +/// Holds a [`Weak`] so the global never keeps the manager alive past its last +/// strong reference: when the final [`Arc`] returned by +/// [`FoundryLocalManager::create`] is dropped, the native manager is torn down +/// deterministically via [`Drop`] — while the ORT runtime is still alive and +/// before the library's C++ static destructors run. +/// +/// Wrapped in a [`OnceLock`] (rather than a `const` `Mutex::new`) to keep the +/// crate compatible with its minimum supported Rust version. +static INSTANCE: OnceLock>> = OnceLock::new(); + +/// The lazily-initialised slot holding the shared-instance weak handle. +fn instance_slot() -> &'static Mutex> { + INSTANCE.get_or_init(|| Mutex::new(Weak::new())) +} + +/// Primary entry point for interacting with Foundry Local. +/// +/// Obtain a handle with [`FoundryLocalManager::create`]. While at least one +/// handle is alive, every caller shares the same instance. +pub struct FoundryLocalManager { + native: Arc, + catalog: Catalog, + urls: Mutex>, + /// Application logger (stub — not yet wired into the native core). + _logger: Option>, +} + +type EpDownloadProgressCallback = Box; + +/// Builder for configuring and running execution provider downloads. +pub struct EpDownloadBuilder<'a> { + manager: &'a FoundryLocalManager, + names: Option>, + progress_callback: Option, + cancel_flag: Option>, +} + +impl<'a> EpDownloadBuilder<'a> { + fn new(manager: &'a FoundryLocalManager) -> Self { + Self { + manager, + names: None, + progress_callback: None, + cancel_flag: None, + } + } + + /// Download only the named execution providers. + pub fn names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.names = Some(names.into_iter().map(Into::into).collect()); + self + } + + /// Report per-EP download progress as `(ep_name, percent)`. + pub fn progress(mut self, callback: F) -> Self + where + F: FnMut(&str, f64) + Send + 'static, + { + self.progress_callback = Some(Box::new(callback)); + self + } + + /// Cancel the download when `cancel_flag` is set to `true`. + pub fn cancel(mut self, cancel_flag: Arc) -> Self { + self.cancel_flag = Some(cancel_flag); + self + } + + /// Run the configured execution provider download. + pub async fn run(self) -> Result { + self.manager + .download_and_register_eps_impl(self.names, self.progress_callback, self.cancel_flag) + .await + } +} + +impl FoundryLocalManager { + /// Initialise the SDK and return a shared handle to the manager. + /// + /// While at least one returned [`Arc`] is alive, every call returns the + /// **same** instance (a process-wide singleton) and the `config` passed to + /// later calls is ignored. The native manager is torn down once every handle + /// derived from it is gone — this `Arc`, plus any [`Model`](crate::Model), + /// client, or session it produced, each of which keeps the native manager + /// alive so handles can safely outlive this `Arc`. A subsequent call then + /// builds a fresh instance from the new `config`. + /// + /// Teardown runs via [`Drop`] when the final handle is released — not via a + /// process-exit hook — so the native manager (and its EP unregistration) + /// shuts down while the engine / ORT runtime is still alive. This matches + /// the C++ SDK's local-`Manager` semantics and avoids the WebGPU + /// `ReleaseEpFactory` teardown throw (ORT #29206). + pub fn create(config: FoundryLocalConfig) -> Result> { + // Hold the lock across initialisation so only one thread builds the + // instance; concurrent callers then observe and share it. A poisoned + // lock is recoverable: the guarded `Weak` is valid regardless of panics. + let mut slot = instance_slot() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + // Reuse the live instance if one already exists. + if let Some(existing) = slot.upgrade() { + return Ok(existing); + } + + let mut config = config; + let api = Arc::new(Api::load(config.library_path_ref())?); + let logger = config.take_logger(); + let native_config = config.build_native(&api)?; + + let native = Arc::new(NativeManager::create( + Arc::clone(&api), + native_config.as_ptr(), + )?); + + let catalog_ptr = native.catalog_ptr()?; + let catalog = Catalog::new(Arc::clone(&api), catalog_ptr, Arc::clone(&native))?; + + let manager = Arc::new(FoundryLocalManager { + native, + catalog, + urls: Mutex::new(Vec::new()), + _logger: logger, + }); + + // Record a weak reference so future calls share this instance without + // keeping it alive past the caller's last strong handle. + *slot = Arc::downgrade(&manager); + Ok(manager) + } + + /// Access the model catalog. + pub fn catalog(&self) -> &Catalog { + &self.catalog + } + + /// Begin a graceful shutdown of the local engine. + /// + /// Stops the web service, prevents new model loads, stops existing + /// sessions, and unloads models. Idempotent and safe to call from any + /// thread. + /// + /// Calling this is optional: the native manager is released automatically + /// when the last handle is dropped. Use it when you want to deterministically + /// wind the engine down before releasing the handle. After calling + /// `shutdown`, the manager should not be used for further inference. + pub fn shutdown(&self) -> Result<()> { + self.native.shutdown() + } + + /// URLs that the local web service is listening on. + /// + /// Empty until [`Self::start_web_service`] has been called. + pub fn urls(&self) -> Result> { + let lock = self.urls.lock().map_err(|_| FoundryLocalError::Internal { + reason: "Failed to acquire urls lock".into(), + })?; + Ok(lock.clone()) + } + + /// Start the local web service. + /// + /// The listening URLs are stored internally and can be retrieved via + /// [`Self::urls`] after this method returns. + pub async fn start_web_service(&self) -> Result<()> { + let native = Arc::clone(&self.native); + let urls = spawn_blocking(move || { + native.web_service_start()?; + native.web_service_urls() + }) + .await?; + *self.urls.lock().map_err(|_| FoundryLocalError::Internal { + reason: "Failed to acquire urls lock".into(), + })? = urls; + Ok(()) + } + + /// Stop the local web service. + pub async fn stop_web_service(&self) -> Result<()> { + let native = Arc::clone(&self.native); + spawn_blocking(move || native.web_service_stop()).await?; + self.urls + .lock() + .map_err(|_| FoundryLocalError::Internal { + reason: "Failed to acquire urls lock".into(), + })? + .clear(); + Ok(()) + } + + /// Discover available execution providers and their registration status. + pub fn discover_eps(&self) -> Result> { + self.native.discover_eps() + } + + /// Download and register execution providers. + /// + /// If `names` is `None` or empty, all available EPs are downloaded. + /// Otherwise only the named EPs are downloaded and registered. + pub async fn download_and_register_eps( + &self, + names: Option<&[&str]>, + ) -> Result { + let names = names.map(|n| n.iter().map(|s| s.to_string()).collect::>()); + self.download_and_register_eps_impl(names, None, None).await + } + + /// Download and register execution providers, reporting per-EP progress. + /// + /// If `names` is `None` or empty, all available EPs are downloaded. + /// Otherwise only the named EPs are downloaded and registered. + /// + /// `progress_callback` receives `(ep_name, percent)` where `percent` + /// ranges from 0.0 to 100.0 as each EP downloads. + pub async fn download_and_register_eps_with_progress( + &self, + names: Option<&[&str]>, + progress_callback: F, + ) -> Result + where + F: FnMut(&str, f64) + Send + 'static, + { + let names = names.map(|n| n.iter().map(|s| s.to_string()).collect::>()); + self.download_and_register_eps_impl(names, Some(Box::new(progress_callback)), None) + .await + } + + /// Configure and run execution provider downloads with a builder. + /// + /// Use this for call sites that need names, progress, cancellation, or + /// future download options. + pub fn download_and_register_eps_builder(&self) -> EpDownloadBuilder<'_> { + EpDownloadBuilder::new(self) + } + + async fn download_and_register_eps_impl( + &self, + names: Option>, + progress_callback: Option, + cancel_flag: Option>, + ) -> Result { + let native = Arc::clone(&self.native); + + // Snapshot requested EP names (default: all discoverable). + let requested: Vec = match &names { + Some(n) if !n.is_empty() => n.clone(), + _ => native + .discover_eps() + .unwrap_or_default() + .into_iter() + .map(|e| e.name) + .collect(), + }; + + let (message, after) = spawn_blocking(move || { + let name_refs: Option> = names + .as_ref() + .map(|n| n.iter().map(String::as_str).collect()); + let progress: Option = + progress_callback.map(|cb| cb as EpProgressCallback); + let message = + native.download_and_register_eps(name_refs.as_deref(), progress, cancel_flag); + // Re-query registration state to synthesise the per-EP result. + let after = native.discover_eps().unwrap_or_default(); + Ok::<(Option, Vec), FoundryLocalError>((message, after)) + }) + .await?; + + let registered_eps: Vec = requested + .iter() + .filter(|name| after.iter().any(|e| &e.name == *name && e.is_registered)) + .cloned() + .collect(); + let failed_eps: Vec = requested + .iter() + .filter(|name| !registered_eps.contains(*name)) + .cloned() + .collect(); + + let success = message.is_none() && failed_eps.is_empty(); + let status = match &message { + None => "All requested execution providers were registered successfully.".to_string(), + Some(msg) if msg.is_empty() => { + "One or more execution providers failed to register.".to_string() + } + Some(msg) => msg.clone(), + }; + + let result = EpDownloadResult { + success, + status, + registered_eps, + failed_eps, + }; + + // Invalidate the catalog cache if any EP was newly registered so the next + // access re-fetches models with the updated set of available EPs. + if result.success || !result.registered_eps.is_empty() { + let _ = self.catalog.update_models().await; + } + + Ok(result) + } +} diff --git a/sdk_v2/rust/src/item.rs b/sdk_v2/rust/src/item.rs new file mode 100644 index 000000000..c6feefdac --- /dev/null +++ b/sdk_v2/rust/src/item.rs @@ -0,0 +1,721 @@ +//! The [`Item`] discriminated union and its supporting value types. +//! +//! An [`Item`] is the unit of data exchanged with a [`Session`](crate::Session): +//! requests carry input items (text, messages, images, audio, tensors, tool +//! calls/results) and responses carry output items (text, tensors, tool calls, +//! speech results, …). +//! +//! Unlike the class hierarchies used by the C++/C#/Python SDKs, the Rust surface +//! models items as a plain, owned `enum`. Items are pure data: they hold no +//! native handle, are `Send + Sync + Clone`, and can be constructed, matched, +//! and inspected without a loaded native library. Conversion to and from the +//! native `flItem` representation happens transiently inside the session when a +//! request is processed (see [`crate::detail::items`]). + +use crate::detail::ffi::*; + +/// The kind of payload carried by an [`Item`]. +/// +/// Mirrors the native `flItemType` discriminant and is returned by +/// [`Item::item_type`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ItemType { + /// Opaque byte buffer ([`Item::Bytes`]). + Bytes, + /// Numeric tensor ([`Item::Tensor`]). + Tensor, + /// UTF-8 text ([`Item::Text`]). + Text, + /// Chat message with nested content parts ([`Item::Message`]). + Message, + /// Image, inline or by URI ([`Item::Image`]). + Image, + /// Audio, inline or by URI ([`Item::Audio`]). + Audio, + /// A single speech-recognition segment ([`Item::SpeechSegment`]). + SpeechSegment, + /// A complete speech-recognition result ([`Item::SpeechResult`]). + SpeechResult, + /// A model-issued tool/function call ([`Item::ToolCall`]). + ToolCall, + /// The result of executing a tool/function call ([`Item::ToolResult`]). + ToolResult, +} + +/// The subtype of a [`Item::Text`] payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum TextKind { + /// Ordinary user- or model-visible text. + #[default] + Default, + /// Model reasoning / chain-of-thought text. + Reasoning, + /// An opaque OpenAI-compatible REST JSON payload (used by the higher-level + /// OpenAI facade). Rarely constructed directly. + OpenAiJson, +} + +impl TextKind { + pub(crate) fn to_native(self) -> flTextItemType { + match self { + TextKind::Default => FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT, + TextKind::Reasoning => FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING, + TextKind::OpenAiJson => FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON, + } + } + + pub(crate) fn from_native(value: flTextItemType) -> TextKind { + match value { + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING => TextKind::Reasoning, + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON => TextKind::OpenAiJson, + _ => TextKind::Default, + } + } +} + +/// The author role of a chat [`Message`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum MessageRole { + /// No role specified (only seen when reading messages that omit a role). + #[default] + None, + /// System / instruction message. + System, + /// End-user message. + User, + /// Model / assistant message. + Assistant, + /// Tool / function output message. + Tool, + /// Developer message (higher-priority instructions than `System`). + Developer, +} + +impl MessageRole { + pub(crate) fn to_native(self) -> flMessageRole { + match self { + MessageRole::None => FOUNDRY_LOCAL_ROLE_NONE, + MessageRole::System => FOUNDRY_LOCAL_ROLE_SYSTEM, + MessageRole::User => FOUNDRY_LOCAL_ROLE_USER, + MessageRole::Assistant => FOUNDRY_LOCAL_ROLE_ASSISTANT, + MessageRole::Tool => FOUNDRY_LOCAL_ROLE_TOOL, + MessageRole::Developer => FOUNDRY_LOCAL_ROLE_DEVELOPER, + } + } + + pub(crate) fn from_native(value: flMessageRole) -> MessageRole { + match value { + FOUNDRY_LOCAL_ROLE_SYSTEM => MessageRole::System, + FOUNDRY_LOCAL_ROLE_USER => MessageRole::User, + FOUNDRY_LOCAL_ROLE_ASSISTANT => MessageRole::Assistant, + FOUNDRY_LOCAL_ROLE_TOOL => MessageRole::Tool, + FOUNDRY_LOCAL_ROLE_DEVELOPER => MessageRole::Developer, + _ => MessageRole::None, + } + } +} + +/// The element data type of a [`Tensor`], mirroring ONNX tensor element types. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[allow(missing_docs)] +pub enum TensorDataType { + #[default] + Undefined, + Float, + Uint8, + Int8, + Uint16, + Int16, + Int32, + Int64, + String, + Bool, + Float16, + Double, + Uint32, + Uint64, + Complex64, + Complex128, + BFloat16, + Float8E4M3FN, + Float8E4M3FNUZ, + Float8E5M2, + Float8E5M2FNUZ, + Uint4, + Int4, + Float4E2M1, + Float8E8M0, +} + +impl TensorDataType { + pub(crate) fn to_native(self) -> flTensorDataType { + match self { + TensorDataType::Undefined => FOUNDRY_LOCAL_TENSOR_UNDEFINED, + TensorDataType::Float => FOUNDRY_LOCAL_TENSOR_FLOAT, + TensorDataType::Uint8 => FOUNDRY_LOCAL_TENSOR_UINT8, + TensorDataType::Int8 => FOUNDRY_LOCAL_TENSOR_INT8, + TensorDataType::Uint16 => FOUNDRY_LOCAL_TENSOR_UINT16, + TensorDataType::Int16 => FOUNDRY_LOCAL_TENSOR_INT16, + TensorDataType::Int32 => FOUNDRY_LOCAL_TENSOR_INT32, + TensorDataType::Int64 => FOUNDRY_LOCAL_TENSOR_INT64, + TensorDataType::String => FOUNDRY_LOCAL_TENSOR_STRING, + TensorDataType::Bool => FOUNDRY_LOCAL_TENSOR_BOOL, + TensorDataType::Float16 => FOUNDRY_LOCAL_TENSOR_FLOAT16, + TensorDataType::Double => FOUNDRY_LOCAL_TENSOR_DOUBLE, + TensorDataType::Uint32 => FOUNDRY_LOCAL_TENSOR_UINT32, + TensorDataType::Uint64 => FOUNDRY_LOCAL_TENSOR_UINT64, + TensorDataType::Complex64 => FOUNDRY_LOCAL_TENSOR_COMPLEX64, + TensorDataType::Complex128 => FOUNDRY_LOCAL_TENSOR_COMPLEX128, + TensorDataType::BFloat16 => FOUNDRY_LOCAL_TENSOR_BFLOAT16, + TensorDataType::Float8E4M3FN => FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FN, + TensorDataType::Float8E4M3FNUZ => FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FNUZ, + TensorDataType::Float8E5M2 => FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2, + TensorDataType::Float8E5M2FNUZ => FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2FNUZ, + TensorDataType::Uint4 => FOUNDRY_LOCAL_TENSOR_UINT4, + TensorDataType::Int4 => FOUNDRY_LOCAL_TENSOR_INT4, + TensorDataType::Float4E2M1 => FOUNDRY_LOCAL_TENSOR_FLOAT4E2M1, + TensorDataType::Float8E8M0 => FOUNDRY_LOCAL_TENSOR_FLOAT8E8M0, + } + } + + pub(crate) fn from_native(value: flTensorDataType) -> TensorDataType { + match value { + FOUNDRY_LOCAL_TENSOR_FLOAT => TensorDataType::Float, + FOUNDRY_LOCAL_TENSOR_UINT8 => TensorDataType::Uint8, + FOUNDRY_LOCAL_TENSOR_INT8 => TensorDataType::Int8, + FOUNDRY_LOCAL_TENSOR_UINT16 => TensorDataType::Uint16, + FOUNDRY_LOCAL_TENSOR_INT16 => TensorDataType::Int16, + FOUNDRY_LOCAL_TENSOR_INT32 => TensorDataType::Int32, + FOUNDRY_LOCAL_TENSOR_INT64 => TensorDataType::Int64, + FOUNDRY_LOCAL_TENSOR_STRING => TensorDataType::String, + FOUNDRY_LOCAL_TENSOR_BOOL => TensorDataType::Bool, + FOUNDRY_LOCAL_TENSOR_FLOAT16 => TensorDataType::Float16, + FOUNDRY_LOCAL_TENSOR_DOUBLE => TensorDataType::Double, + FOUNDRY_LOCAL_TENSOR_UINT32 => TensorDataType::Uint32, + FOUNDRY_LOCAL_TENSOR_UINT64 => TensorDataType::Uint64, + FOUNDRY_LOCAL_TENSOR_COMPLEX64 => TensorDataType::Complex64, + FOUNDRY_LOCAL_TENSOR_COMPLEX128 => TensorDataType::Complex128, + FOUNDRY_LOCAL_TENSOR_BFLOAT16 => TensorDataType::BFloat16, + FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FN => TensorDataType::Float8E4M3FN, + FOUNDRY_LOCAL_TENSOR_FLOAT8E4M3FNUZ => TensorDataType::Float8E4M3FNUZ, + FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2 => TensorDataType::Float8E5M2, + FOUNDRY_LOCAL_TENSOR_FLOAT8E5M2FNUZ => TensorDataType::Float8E5M2FNUZ, + FOUNDRY_LOCAL_TENSOR_UINT4 => TensorDataType::Uint4, + FOUNDRY_LOCAL_TENSOR_INT4 => TensorDataType::Int4, + FOUNDRY_LOCAL_TENSOR_FLOAT4E2M1 => TensorDataType::Float4E2M1, + FOUNDRY_LOCAL_TENSOR_FLOAT8E8M0 => TensorDataType::Float8E8M0, + _ => TensorDataType::Undefined, + } + } +} + +/// Whether a [`SpeechSegment`] is an interim hypothesis or a stable result. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum SpeechSegmentKind { + /// Unspecified. + #[default] + None, + /// An interim hypothesis that may still change. + Partial, + /// A stabilized, final segment. + Final, +} + +impl SpeechSegmentKind { + pub(crate) fn from_native(value: flSpeechSegmentKind) -> SpeechSegmentKind { + match value { + FOUNDRY_LOCAL_SPEECH_SEGMENT_PARTIAL => SpeechSegmentKind::Partial, + FOUNDRY_LOCAL_SPEECH_SEGMENT_FINAL => SpeechSegmentKind::Final, + _ => SpeechSegmentKind::None, + } + } +} + +/// The source of an [`Image`] or [`Audio`] payload: inline bytes or a URI. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MediaSource { + /// Inline, raw bytes. + Data(Vec), + /// A reference to external content by URI. + Uri(String), +} + +/// A chat message: an author [`role`](MessageRole) plus ordered content parts. +/// +/// Content parts are themselves [`Item`]s, allowing multimodal messages (e.g. a +/// [`Item::Text`] alongside a [`Item::Image`]). +#[derive(Debug, Clone, PartialEq)] +pub struct Message { + /// The author role. + pub role: MessageRole, + /// The ordered content parts of the message. + pub content: Vec, + /// Optional participant name. + pub name: Option, +} + +impl Message { + /// Create a message with the given role and content parts. + pub fn new(role: MessageRole, content: impl Into>) -> Self { + Self { + role, + content: content.into(), + name: None, + } + } + + /// Set the participant name (builder-style). + pub fn with_name(mut self, name: impl Into) -> Self { + self.name = Some(name.into()); + self + } + + /// Whether the message consists of a single [`Item::Text`] content part. + pub fn is_simple_text(&self) -> bool { + matches!(self.content.as_slice(), [Item::Text { .. }]) + } + + /// The concatenated text of all [`Item::Text`] content parts. + /// + /// Non-text parts are ignored. Returns an empty string if there are none. + pub fn text(&self) -> String { + let mut out = String::new(); + for part in &self.content { + if let Item::Text { text, .. } = part { + out.push_str(text); + } + } + out + } +} + +/// A numeric tensor: an element [`data_type`](TensorDataType), a `shape`, and the +/// raw little-endian element bytes. +#[derive(Debug, Clone, PartialEq)] +pub struct Tensor { + /// The element data type. + pub data_type: TensorDataType, + /// The dimensions of the tensor. + pub shape: Vec, + /// The raw element bytes, row-major, little-endian. + pub data: Vec, +} + +impl Tensor { + /// Interpret the raw bytes as `f32` elements. + /// + /// Returns `None` unless [`data_type`](Self::data_type) is + /// [`TensorDataType::Float`] and the byte length is a multiple of four. + pub fn as_f32(&self) -> Option> { + if self.data_type != TensorDataType::Float || self.data.len() % 4 != 0 { + return None; + } + Some( + self.data + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(), + ) + } +} + +/// An image payload, inline or by URI, with an optional `format` hint (e.g. `png`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Image { + /// The image content. + pub source: MediaSource, + /// Optional format hint (e.g. `"png"`, `"jpeg"`). + pub format: Option, +} + +/// An audio payload, inline or by URI, with format and PCM layout hints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Audio { + /// The audio content. + pub source: MediaSource, + /// Optional format hint (e.g. `"wav"`, `"pcm16"`). + pub format: Option, + /// Sample rate in Hz, or `0` if unspecified. + pub sample_rate: i32, + /// Channel count, or `0` if unspecified. + pub channels: i32, +} + +/// A model-issued request to invoke a named tool/function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ToolCall { + /// Correlates the call with its [`ToolResult`]. + pub call_id: String, + /// The name of the tool/function to invoke. + pub name: String, + /// The call arguments, as a JSON string. + pub arguments: String, +} + +/// The result of executing a [`ToolCall`], fed back to the model. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ToolResult { + /// The `call_id` of the originating [`ToolCall`]. + pub call_id: String, + /// The tool output, typically a JSON or text string. + pub result: String, +} + +/// One recognized word within a [`SpeechSegment`] (output-only). +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechWord { + /// The word text. + pub text: String, + /// Start offset from the beginning of the audio, in milliseconds. + pub start_time_ms: Option, + /// End offset from the beginning of the audio, in milliseconds. + pub end_time_ms: Option, + /// Recognition confidence in `[0, 1]`, if reported. + pub confidence: Option, + /// Diarization speaker id, if reported. + pub speaker_id: Option, +} + +/// One speech-recognition segment (output-only). +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechSegment { + /// Whether this is an interim or final segment. + pub kind: SpeechSegmentKind, + /// The recognized text of the segment. + pub text: String, + /// Start offset from the beginning of the audio, in milliseconds. + pub start_time_ms: Option, + /// End offset from the beginning of the audio, in milliseconds. + pub end_time_ms: Option, + /// Whether this segment starts a new utterance. + pub utterance_start: bool, + /// Per-word timing/confidence, when available. + pub words: Vec, + /// Detected language (e.g. `"en"`), if reported. + pub language: Option, +} + +/// A complete speech-recognition result (output-only). +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechResult { + /// The full concatenated transcript. + pub text: String, + /// Detected language (e.g. `"en"`), if reported. + pub language: Option, + /// Total audio duration in milliseconds, if reported. + pub duration_ms: Option, + /// The constituent segments, each a [`Item::SpeechSegment`]. + pub segments: Vec, +} + +/// A single unit of input or output data exchanged with a [`Session`](crate::Session). +/// +/// `Item` is a pure-data, owned value: it holds no native handle and is +/// `Send + Sync + Clone`. Construct items with the associated functions (e.g. +/// [`Item::text`], [`Item::user_message`], [`Item::image_data`]) or the struct +/// variants directly, and inspect them by pattern matching. +#[derive(Debug, Clone, PartialEq)] +pub enum Item { + /// UTF-8 text of a given [`TextKind`]. + Text { + /// The text content. + text: String, + /// The text subtype. + kind: TextKind, + }, + /// A chat message with nested content parts. + Message(Message), + /// An opaque byte buffer. + Bytes(Vec), + /// A numeric tensor (e.g. an embedding vector). + Tensor(Tensor), + /// An image, inline or by URI. + Image(Image), + /// An audio clip, inline or by URI. + Audio(Audio), + /// A model-issued tool/function call. + ToolCall(ToolCall), + /// The result of executing a tool/function call. + ToolResult(ToolResult), + /// A speech-recognition segment (output-only). + SpeechSegment(SpeechSegment), + /// A speech-recognition result (output-only). + SpeechResult(SpeechResult), +} + +impl Item { + /// The [`ItemType`] discriminant of this item. + pub fn item_type(&self) -> ItemType { + match self { + Item::Text { .. } => ItemType::Text, + Item::Message(_) => ItemType::Message, + Item::Bytes(_) => ItemType::Bytes, + Item::Tensor(_) => ItemType::Tensor, + Item::Image(_) => ItemType::Image, + Item::Audio(_) => ItemType::Audio, + Item::ToolCall(_) => ItemType::ToolCall, + Item::ToolResult(_) => ItemType::ToolResult, + Item::SpeechSegment(_) => ItemType::SpeechSegment, + Item::SpeechResult(_) => ItemType::SpeechResult, + } + } + + // ── Text constructors ──────────────────────────────────────────────────── + + /// A default-kind [`Item::Text`]. + pub fn text(text: impl Into) -> Self { + Item::Text { + text: text.into(), + kind: TextKind::Default, + } + } + + /// A reasoning / chain-of-thought [`Item::Text`]. + pub fn reasoning(text: impl Into) -> Self { + Item::Text { + text: text.into(), + kind: TextKind::Reasoning, + } + } + + // ── Message constructors ───────────────────────────────────────────────── + + /// A [`Item::Message`] with the given role and content parts. + pub fn message(role: MessageRole, content: impl Into>) -> Self { + Item::Message(Message::new(role, content)) + } + + /// A `system` [`Item::Message`]. + pub fn system_message(content: impl Into>) -> Self { + Item::message(MessageRole::System, content) + } + + /// A `user` [`Item::Message`]. + pub fn user_message(content: impl Into>) -> Self { + Item::message(MessageRole::User, content) + } + + /// An `assistant` [`Item::Message`]. + pub fn assistant_message(content: impl Into>) -> Self { + Item::message(MessageRole::Assistant, content) + } + + /// A `developer` [`Item::Message`]. + pub fn developer_message(content: impl Into>) -> Self { + Item::message(MessageRole::Developer, content) + } + + /// A `tool` [`Item::Message`]. + pub fn tool_message(content: impl Into>) -> Self { + Item::message(MessageRole::Tool, content) + } + + // ── Binary / tensor constructors ───────────────────────────────────────── + + /// An [`Item::Bytes`] carrying an opaque byte buffer. + pub fn bytes(data: impl Into>) -> Self { + Item::Bytes(data.into()) + } + + /// An [`Item::Tensor`] from a data type, shape, and raw element bytes. + pub fn tensor( + data_type: TensorDataType, + shape: impl Into>, + data: impl Into>, + ) -> Self { + Item::Tensor(Tensor { + data_type, + shape: shape.into(), + data: data.into(), + }) + } + + /// A [`TensorDataType::Float`] [`Item::Tensor`] from `f32` elements. + pub fn float_tensor(shape: impl Into>, data: &[f32]) -> Self { + let mut bytes = Vec::with_capacity(data.len() * 4); + for f in data { + bytes.extend_from_slice(&f.to_le_bytes()); + } + Item::Tensor(Tensor { + data_type: TensorDataType::Float, + shape: shape.into(), + data: bytes, + }) + } + + // ── Image / audio constructors ─────────────────────────────────────────── + + /// An [`Item::Image`] from inline bytes and an optional format hint. + pub fn image_data(data: impl Into>, format: Option>) -> Self { + Item::Image(Image { + source: MediaSource::Data(data.into()), + format: format.map(Into::into), + }) + } + + /// An [`Item::Image`] referencing external content by URI. + pub fn image_uri(uri: impl Into, format: Option>) -> Self { + Item::Image(Image { + source: MediaSource::Uri(uri.into()), + format: format.map(Into::into), + }) + } + + /// An [`Item::Audio`] from inline bytes, format hint, and PCM layout. + pub fn audio_data( + data: impl Into>, + format: Option>, + sample_rate: i32, + channels: i32, + ) -> Self { + Item::Audio(Audio { + source: MediaSource::Data(data.into()), + format: format.map(Into::into), + sample_rate, + channels, + }) + } + + /// An [`Item::Audio`] referencing external content by URI. + pub fn audio_uri( + uri: impl Into, + format: Option>, + sample_rate: i32, + channels: i32, + ) -> Self { + Item::Audio(Audio { + source: MediaSource::Uri(uri.into()), + format: format.map(Into::into), + sample_rate, + channels, + }) + } + + // ── Tool constructors ──────────────────────────────────────────────────── + + /// A [`Item::ToolCall`]. + pub fn tool_call( + call_id: impl Into, + name: impl Into, + arguments: impl Into, + ) -> Self { + Item::ToolCall(ToolCall { + call_id: call_id.into(), + name: name.into(), + arguments: arguments.into(), + }) + } + + /// A [`Item::ToolResult`]. + pub fn tool_result(call_id: impl Into, result: impl Into) -> Self { + Item::ToolResult(ToolResult { + call_id: call_id.into(), + result: result.into(), + }) + } + + // ── Accessors ──────────────────────────────────────────────────────────── + + /// The text content, if this is a [`Item::Text`]. + pub fn as_text(&self) -> Option<&str> { + match self { + Item::Text { text, .. } => Some(text), + _ => None, + } + } + + /// The [`Message`], if this is a [`Item::Message`]. + pub fn as_message(&self) -> Option<&Message> { + match self { + Item::Message(m) => Some(m), + _ => None, + } + } + + /// The [`Tensor`], if this is a [`Item::Tensor`]. + pub fn as_tensor(&self) -> Option<&Tensor> { + match self { + Item::Tensor(t) => Some(t), + _ => None, + } + } + + /// The [`ToolCall`], if this is a [`Item::ToolCall`]. + pub fn as_tool_call(&self) -> Option<&ToolCall> { + match self { + Item::ToolCall(c) => Some(c), + _ => None, + } + } + + /// The [`SpeechResult`], if this is a [`Item::SpeechResult`]. + pub fn as_speech_result(&self) -> Option<&SpeechResult> { + match self { + Item::SpeechResult(r) => Some(r), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn item_type_matches_variant() { + assert_eq!(Item::text("hi").item_type(), ItemType::Text); + assert_eq!(Item::bytes(vec![1, 2]).item_type(), ItemType::Bytes); + assert_eq!( + Item::tool_call("c", "f", "{}").item_type(), + ItemType::ToolCall + ); + } + + #[test] + fn message_helpers() { + let m = Item::user_message(vec![Item::text("hello"), Item::text(" world")]); + let msg = m.as_message().unwrap(); + assert_eq!(msg.role, MessageRole::User); + assert!(!msg.is_simple_text()); + assert_eq!(msg.text(), "hello world"); + + let simple = Message::new(MessageRole::System, vec![Item::text("x")]); + assert!(simple.is_simple_text()); + } + + #[test] + fn float_tensor_round_trips_bytes() { + let values = [1.0f32, -2.5, 3.25]; + let item = Item::float_tensor(vec![3], &values); + let t = item.as_tensor().unwrap(); + assert_eq!(t.data_type, TensorDataType::Float); + assert_eq!(t.shape, vec![3]); + assert_eq!(t.as_f32().unwrap(), values); + } + + #[test] + fn native_enum_mappings_round_trip() { + for kind in [TextKind::Default, TextKind::Reasoning, TextKind::OpenAiJson] { + assert_eq!(TextKind::from_native(kind.to_native()), kind); + } + for role in [ + MessageRole::None, + MessageRole::System, + MessageRole::User, + MessageRole::Assistant, + MessageRole::Tool, + MessageRole::Developer, + ] { + assert_eq!(MessageRole::from_native(role.to_native()), role); + } + for dt in [ + TensorDataType::Undefined, + TensorDataType::Float, + TensorDataType::Int64, + TensorDataType::BFloat16, + TensorDataType::Float8E8M0, + ] { + assert_eq!(TensorDataType::from_native(dt.to_native()), dt); + } + } +} diff --git a/sdk_v2/rust/src/item_queue.rs b/sdk_v2/rust/src/item_queue.rs new file mode 100644 index 000000000..e1c15535c --- /dev/null +++ b/sdk_v2/rust/src/item_queue.rs @@ -0,0 +1,79 @@ +//! The [`ItemQueue`] handle: a thread-safe stream of [`Item`]s. +//! +//! Unlike [`Item`]/[`Request`](crate::Request)/[`Response`](crate::Response), +//! `ItemQueue` is *handle-backed*: it wraps a native `flItemQueue` and shares it +//! across clones (an `Arc`), because the queue has a lifetime and identity that +//! must be preserved. Create one from a [`Session`](crate::Session) via +//! [`Session::create_input_queue`](crate::Session::create_input_queue) and attach +//! it to a [`Request`](crate::Request) for incremental / streaming input. + +use std::sync::Arc; + +use crate::detail::session::NativeItemQueue; +use crate::error::Result; +use crate::item::Item; + +/// A thread-safe, multi-producer / multi-consumer queue of [`Item`]s. +/// +/// Cloning an `ItemQueue` yields another handle to the *same* underlying native +/// queue, so items pushed through one clone are visible to all. +#[derive(Clone)] +pub struct ItemQueue { + inner: Arc, +} + +impl ItemQueue { + pub(crate) fn from_native(inner: Arc) -> Self { + Self { inner } + } + + /// Borrow the underlying native queue (for request wiring within the crate). + pub(crate) fn native(&self) -> &NativeItemQueue { + &self.inner + } + + /// Consume this handle, yielding the shared native queue. + pub(crate) fn into_native(self) -> Arc { + self.inner + } + + /// Push an item onto the queue, transferring a native copy into it. + pub fn push(&self, item: &Item) -> Result<()> { + self.inner.push_value(item) + } + + /// Pop the next available item, or `None` if the queue is currently empty. + pub fn try_pop(&self) -> Option { + self.inner.try_pop_value() + } + + /// The number of items currently buffered. + pub fn len(&self) -> usize { + self.inner.size() + } + + /// Whether the queue currently holds no items. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Signal that no more items will be pushed. A consumer draining the queue + /// can then stop once it is empty. + pub fn mark_finished(&self) { + self.inner.mark_finished(); + } + + /// Whether [`mark_finished`](Self::mark_finished) has been called. + pub fn is_finished(&self) -> bool { + self.inner.is_finished() + } +} + +impl std::fmt::Debug for ItemQueue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ItemQueue") + .field("len", &self.len()) + .field("finished", &self.is_finished()) + .finish() + } +} diff --git a/sdk_v2/rust/src/lib.rs b/sdk_v2/rust/src/lib.rs new file mode 100644 index 000000000..dd8a9d12e --- /dev/null +++ b/sdk_v2/rust/src/lib.rs @@ -0,0 +1,64 @@ +//! Foundry Local Rust SDK +//! +//! Local AI model inference powered by the Foundry Local Core engine. + +mod catalog; +mod configuration; +mod error; +mod foundry_local_manager; +mod item; +mod item_queue; +mod request; +mod response; +mod session; +mod types; + +pub(crate) mod detail; +pub mod openai; + +pub use self::catalog::Catalog; +pub use self::configuration::{FoundryLocalConfig, LogLevel, Logger}; +pub use self::detail::model::{DownloadBuilder, Model}; +pub use self::error::FoundryLocalError; +pub use self::foundry_local_manager::{EpDownloadBuilder, FoundryLocalManager}; +pub use self::item::{ + Audio, Image, Item, ItemType, MediaSource, Message, MessageRole, SpeechResult, SpeechSegment, + SpeechSegmentKind, SpeechWord, Tensor, TensorDataType, TextKind, ToolCall, ToolResult, +}; +pub use self::item_queue::ItemQueue; +pub use self::request::{Request, RequestOptions, SearchOptions, ToolChoice}; +pub use self::response::{FinishReason, Response, Usage}; +pub use self::session::{ + AudioSession, ChatSession, EmbeddingsSession, ItemStream, Session, ToolDefinition, +}; +pub use self::types::{ + ChatResponseFormat, ChatToolChoice, DeviceType, EpDownloadResult, EpInfo, ModelInfo, + ModelSettings, Parameter, PromptTemplate, Runtime, +}; + +// Re-export OpenAI request types so callers can construct typed messages. +pub use async_openai::types::chat::{ + ChatCompletionNamedToolChoice, ChatCompletionRequestAssistantMessage, + ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, + ChatCompletionRequestToolMessage, ChatCompletionRequestUserMessage, + ChatCompletionToolChoiceOption, ChatCompletionTools, FunctionObject, +}; + +// Re-export OpenAI response types for convenience. +#[allow(deprecated)] // re-export includes the deprecated `LiveAudioTranscriptionSession` +pub use crate::openai::{ + AudioTranscriptionResponse, AudioTranscriptionStream, ChatCompletionStream, ContentPart, + CoreErrorResponse, LiveAudioTranscriptionOptions, LiveAudioTranscriptionResponse, + LiveAudioTranscriptionSession, LiveAudioTranscriptionStream, TranscriptionSegment, + TranscriptionWord, +}; +// Re-export OpenAI response types for convenience. The OpenAI `FinishReason` is +// re-exported as `ChatFinishReason` to avoid colliding with the core inference +// [`FinishReason`]; it remains available as `async_openai::types::chat::FinishReason`. +pub use async_openai::types::chat::{ + ChatChoice, ChatChoiceStream, ChatCompletionMessageToolCall, + ChatCompletionMessageToolCallChunk, ChatCompletionMessageToolCalls, + ChatCompletionResponseMessage, ChatCompletionStreamResponseDelta, CompletionUsage, + CreateChatCompletionResponse, CreateChatCompletionStreamResponse, + FinishReason as ChatFinishReason, FunctionCall, FunctionCallStream, +}; diff --git a/sdk_v2/rust/src/openai/audio_client.rs b/sdk_v2/rust/src/openai/audio_client.rs new file mode 100644 index 000000000..04133990e --- /dev/null +++ b/sdk_v2/rust/src/openai/audio_client.rs @@ -0,0 +1,222 @@ +//! OpenAI-compatible audio transcription client. +#![allow(deprecated)] // this module implements the deprecated OpenAI facade + +use std::path::Path; + +use serde_json::{json, Value}; + +use crate::detail::native::NativeModel; +use crate::detail::session::{run_openai_json_streaming, NativeSession}; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; + +use super::json_stream::JsonStream; +use super::live_audio_session::LiveAudioTranscriptionSession; + +/// A segment of a transcription, as returned by the OpenAI-compatible API. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct TranscriptionSegment { + /// Segment index. + pub id: i32, + /// Seek offset of the segment. + pub seek: i32, + /// Start time of the segment in seconds. + pub start: f64, + /// End time of the segment in seconds. + pub end: f64, + /// Transcribed text of the segment. + pub text: String, + /// Token IDs corresponding to the text. + #[serde(skip_serializing_if = "Option::is_none")] + pub tokens: Option>, + /// Temperature used for generation. + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + /// Average log probability of the segment. + #[serde(skip_serializing_if = "Option::is_none")] + pub avg_logprob: Option, + /// Compression ratio of the segment. + #[serde(skip_serializing_if = "Option::is_none")] + pub compression_ratio: Option, + /// Probability of no speech in the segment. + #[serde(skip_serializing_if = "Option::is_none")] + pub no_speech_prob: Option, +} + +/// A word with timing information, as returned by the OpenAI-compatible API. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct TranscriptionWord { + /// The word text. + pub word: String, + /// Start time of the word in seconds. + pub start: f64, + /// End time of the word in seconds. + pub end: f64, +} + +/// OpenAI-compatible audio transcription response. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct AudioTranscriptionResponse { + /// The transcribed text. + pub text: String, + /// The language of the input audio (if detected). + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + /// Duration of the input audio in seconds (if available). + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + /// Segments of the transcription (if available). + #[serde(skip_serializing_if = "Option::is_none")] + pub segments: Option>, + /// Words with timestamps (if available). + #[serde(skip_serializing_if = "Option::is_none")] + pub words: Option>, +} + +/// Tuning knobs for audio transcription requests. +/// +/// Use the chainable setter methods to configure, e.g.: +/// +/// ```ignore +/// let client = model.create_audio_client() +/// .language("en") +/// .temperature(0.2); +/// ``` +#[derive(Debug, Clone, Default)] +pub struct AudioClientSettings { + language: Option, + temperature: Option, +} + +impl AudioClientSettings { + fn serialize(&self, model_id: &str, file_name: &str) -> Value { + let mut map = serde_json::Map::new(); + + map.insert("model".into(), json!(model_id)); + map.insert("filename".into(), json!(file_name)); + + if let Some(ref lang) = self.language { + map.insert("language".into(), json!(lang)); + } + if let Some(temp) = self.temperature { + map.insert("temperature".into(), json!(temp)); + } + + Value::Object(map) + } +} + +/// A stream of [`AudioTranscriptionResponse`] chunks. +/// +/// Returned by [`AudioClient::transcribe_streaming`]. +pub type AudioTranscriptionStream = JsonStream; + +/// Client for OpenAI-compatible audio transcription backed by a local model. +#[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use the Session API instead \ + (`AudioSession::new(&model)`)." +)] +pub struct AudioClient { + model_id: String, + model: NativeModel, + settings: AudioClientSettings, +} + +impl AudioClient { + pub(crate) fn new(model_id: &str, model: NativeModel) -> Self { + Self { + model_id: model_id.to_owned(), + model, + settings: AudioClientSettings::default(), + } + } + + /// Set the language hint for transcription. + pub fn language(mut self, lang: impl Into) -> Self { + self.settings.language = Some(lang.into()); + self + } + + /// Set the sampling temperature. + pub fn temperature(mut self, v: f64) -> Self { + self.settings.temperature = Some(v); + self + } + + /// Transcribe an audio file. + pub async fn transcribe( + &self, + audio_file_path: impl AsRef, + ) -> Result { + let path_str = + audio_file_path + .as_ref() + .to_str() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "audio file path is not valid UTF-8".into(), + })?; + Self::validate_path(path_str)?; + + let request = self.settings.serialize(&self.model_id, path_str); + let request_json = serde_json::to_string(&request)?; + let model = self.model.clone(); + + let raw = spawn_blocking(move || { + let session = NativeSession::create(&model)?; + session.run_openai_json(&request_json) + }) + .await?; + + let parsed: AudioTranscriptionResponse = serde_json::from_str(&raw)?; + Ok(parsed) + } + + /// Transcribe an audio file with streaming results, returning an + /// [`AudioTranscriptionStream`]. + pub async fn transcribe_streaming( + &self, + audio_file_path: impl AsRef, + ) -> Result { + let path_str = + audio_file_path + .as_ref() + .to_str() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "audio file path is not valid UTF-8".into(), + })?; + Self::validate_path(path_str)?; + + let request = self.settings.serialize(&self.model_id, path_str); + let request_json = serde_json::to_string(&request)?; + let model = self.model.clone(); + + let session = spawn_blocking(move || NativeSession::create(&model)).await?; + let rx = run_openai_json_streaming(session, request_json, Box::new(Some)); + Ok(AudioTranscriptionStream::new(rx)) + } + + /// Create a [`LiveAudioTranscriptionSession`] for real-time audio + /// streaming transcription. + /// + /// Configure the session's [`settings`](LiveAudioTranscriptionSession::settings) + /// before calling [`start`](LiveAudioTranscriptionSession::start). + #[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use `AudioSession::new(&model)` \ + with streaming instead." + )] + #[allow(deprecated)] + pub fn create_live_transcription_session(&self) -> LiveAudioTranscriptionSession { + LiveAudioTranscriptionSession::new(&self.model_id, self.model.clone()) + } + + fn validate_path(path: &str) -> Result<()> { + if path.trim().is_empty() { + return Err(FoundryLocalError::Validation { + reason: "audio_file_path must be a non-empty string".into(), + }); + } + Ok(()) + } +} diff --git a/sdk_v2/rust/src/openai/chat_client.rs b/sdk_v2/rust/src/openai/chat_client.rs new file mode 100644 index 000000000..e0f5e9262 --- /dev/null +++ b/sdk_v2/rust/src/openai/chat_client.rs @@ -0,0 +1,402 @@ +//! OpenAI-compatible chat completions client. +#![allow(deprecated)] // this module implements the deprecated OpenAI facade + +use std::collections::HashMap; + +use async_openai::types::chat::{ + ChatCompletionRequestMessage, ChatCompletionTools, CreateChatCompletionResponse, + CreateChatCompletionStreamResponse, +}; +use serde_json::{json, Value}; + +use crate::detail::native::NativeModel; +use crate::detail::session::{run_openai_json_streaming, NativeSession}; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; +use crate::types::{ChatResponseFormat, ChatToolChoice}; + +use super::json_stream::JsonStream; + +/// Tuning knobs for chat completion requests. +/// +/// Use the chainable setter methods to configure, e.g.: +/// +/// ```ignore +/// let client = model.create_chat_client() +/// .temperature(0.7) +/// .max_tokens(256); +/// ``` +#[derive(Debug, Clone, Default)] +pub struct ChatClientSettings { + frequency_penalty: Option, + max_tokens: Option, + n: Option, + temperature: Option, + presence_penalty: Option, + top_p: Option, + top_k: Option, + random_seed: Option, + response_format: Option, + tool_choice: Option, +} + +impl ChatClientSettings { + fn serialize(&self) -> Value { + let mut map = serde_json::Map::new(); + + if let Some(v) = self.frequency_penalty { + map.insert("frequency_penalty".into(), json!(v)); + } + if let Some(v) = self.max_tokens { + map.insert("max_tokens".into(), json!(v)); + } + if let Some(v) = self.n { + map.insert("n".into(), json!(v)); + } + if let Some(v) = self.presence_penalty { + map.insert("presence_penalty".into(), json!(v)); + } + if let Some(v) = self.temperature { + map.insert("temperature".into(), json!(v)); + } + if let Some(v) = self.top_p { + map.insert("top_p".into(), json!(v)); + } + + if let Some(ref rf) = self.response_format { + let mut rf_map = serde_json::Map::new(); + match rf { + ChatResponseFormat::Text => { + rf_map.insert("type".into(), json!("text")); + } + ChatResponseFormat::JsonObject => { + rf_map.insert("type".into(), json!("json_object")); + } + ChatResponseFormat::JsonSchema(schema) => { + rf_map.insert("type".into(), json!("json_schema")); + rf_map.insert("json_schema".into(), json!(schema)); + } + ChatResponseFormat::LarkGrammar(grammar) => { + rf_map.insert("type".into(), json!("lark_grammar")); + rf_map.insert("lark_grammar".into(), json!(grammar)); + } + } + map.insert("response_format".into(), Value::Object(rf_map)); + } + + if let Some(ref tc) = self.tool_choice { + // Match the native `*-json` contract (see MapGuidance / + // chat_completions_converter.cc): `none`/`auto`/`required` are plain + // strings, while a named function is `{"type":"function", + // "function":{"name":"…"}}`. + let tc_value = match tc { + ChatToolChoice::None => json!("none"), + ChatToolChoice::Auto => json!("auto"), + ChatToolChoice::Required => json!("required"), + ChatToolChoice::Function(name) => json!({ + "type": "function", + "function": { "name": name }, + }), + }; + map.insert("tool_choice".into(), tc_value); + } + + // Foundry-specific metadata for settings that don't map directly to + // the OpenAI spec. + let mut metadata: HashMap = HashMap::new(); + if let Some(k) = self.top_k { + metadata.insert("top_k".into(), k.to_string()); + } + if let Some(s) = self.random_seed { + metadata.insert("random_seed".into(), s.to_string()); + } + if !metadata.is_empty() { + map.insert("metadata".into(), json!(metadata)); + } + + Value::Object(map) + } +} + +/// A stream of [`CreateChatCompletionStreamResponse`] chunks. +/// +/// Returned by [`ChatClient::complete_streaming_chat`]. +pub type ChatCompletionStream = JsonStream; + +/// Client for OpenAI-compatible chat completions backed by a local model. +#[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use the Session API instead \ + (`ChatSession::new(&model)`)." +)] +pub struct ChatClient { + model_id: String, + model: NativeModel, + settings: ChatClientSettings, +} + +impl ChatClient { + pub(crate) fn new(model_id: &str, model: NativeModel) -> Self { + Self { + model_id: model_id.to_owned(), + model, + settings: ChatClientSettings::default(), + } + } + + /// Set the frequency penalty. + pub fn frequency_penalty(mut self, v: f64) -> Self { + self.settings.frequency_penalty = Some(v); + self + } + + /// Set the maximum number of tokens to generate. + pub fn max_tokens(mut self, v: u32) -> Self { + self.settings.max_tokens = Some(v); + self + } + + /// Set the number of completions to generate. + pub fn n(mut self, v: u32) -> Self { + self.settings.n = Some(v); + self + } + + /// Set the sampling temperature. + pub fn temperature(mut self, v: f64) -> Self { + self.settings.temperature = Some(v); + self + } + + /// Set the presence penalty. + pub fn presence_penalty(mut self, v: f64) -> Self { + self.settings.presence_penalty = Some(v); + self + } + + /// Set the nucleus sampling probability. + pub fn top_p(mut self, v: f64) -> Self { + self.settings.top_p = Some(v); + self + } + + /// Set the top-k sampling parameter (Foundry extension). + pub fn top_k(mut self, v: u32) -> Self { + self.settings.top_k = Some(v); + self + } + + /// Set the random seed for reproducible results (Foundry extension). + pub fn random_seed(mut self, v: u64) -> Self { + self.settings.random_seed = Some(v); + self + } + + /// Set the desired response format. + pub fn response_format(mut self, v: ChatResponseFormat) -> Self { + self.settings.response_format = Some(v); + self + } + + /// Set the tool choice strategy. + pub fn tool_choice(mut self, v: ChatToolChoice) -> Self { + self.settings.tool_choice = Some(v); + self + } + + /// Perform a non-streaming chat completion. + pub async fn complete_chat( + &self, + messages: &[ChatCompletionRequestMessage], + tools: Option<&[ChatCompletionTools]>, + ) -> Result { + if messages.is_empty() { + return Err(FoundryLocalError::Validation { + reason: "messages must be a non-empty array".into(), + }); + } + + let request = self.build_request(messages, tools, false)?; + let request_json = serde_json::to_string(&request)?; + let model = self.model.clone(); + + let raw = spawn_blocking(move || { + let session = NativeSession::create(&model)?; + session.run_openai_json(&request_json) + }) + .await?; + + let parsed: CreateChatCompletionResponse = serde_json::from_str(&raw)?; + Ok(parsed) + } + + /// Perform a streaming chat completion, returning a [`ChatCompletionStream`]. + /// + /// Use the stream with `futures_core::StreamExt::next()` or + /// `tokio_stream::StreamExt::next()`. + pub async fn complete_streaming_chat( + &self, + messages: &[ChatCompletionRequestMessage], + tools: Option<&[ChatCompletionTools]>, + ) -> Result { + if messages.is_empty() { + return Err(FoundryLocalError::Validation { + reason: "messages must be a non-empty array".into(), + }); + } + + let request = self.build_request(messages, tools, true)?; + let request_json = serde_json::to_string(&request)?; + let model = self.model.clone(); + + let session = spawn_blocking(move || NativeSession::create(&model)).await?; + let rx = run_openai_json_streaming(session, request_json, Box::new(normalize_chat_chunk)); + Ok(ChatCompletionStream::new(rx)) + } + + fn build_request( + &self, + messages: &[ChatCompletionRequestMessage], + tools: Option<&[ChatCompletionTools]>, + stream: bool, + ) -> Result { + let settings_value = self.settings.serialize(); + let mut map = match settings_value { + Value::Object(m) => m, + _ => serde_json::Map::new(), + }; + + map.insert("model".into(), json!(self.model_id)); + map.insert("messages".into(), serde_json::to_value(messages)?); + + if stream { + map.insert("stream".into(), json!(true)); + } + + if let Some(t) = tools { + map.insert("tools".into(), serde_json::to_value(t)?); + } + + Ok(Value::Object(map)) + } +} + +/// Normalize a streamed chat chunk so it parses as a +/// [`CreateChatCompletionStreamResponse`]. +/// +/// Foundry Local streams tool calls under `"message"` instead of the standard +/// `"delta"`; rewrite each such choice and ensure tool calls carry an `index`. +/// Chunks that are not valid JSON are passed through unchanged so the stream +/// surfaces the original parse error. +fn normalize_chat_chunk(text: String) -> Option { + let mut value: Value = match serde_json::from_str(&text) { + Ok(v) => v, + Err(_) => return Some(text), + }; + + if let Some(choices) = value.get_mut("choices").and_then(Value::as_array_mut) { + for choice in choices { + let Some(obj) = choice.as_object_mut() else { + continue; + }; + if obj.contains_key("message") && !obj.contains_key("delta") { + if let Some(mut message) = obj.remove("message") { + if let Some(tool_calls) = + message.get_mut("tool_calls").and_then(Value::as_array_mut) + { + for (i, tc) in tool_calls.iter_mut().enumerate() { + if let Some(tc_obj) = tc.as_object_mut() { + tc_obj.entry("index").or_insert_with(|| json!(i)); + } + } + } + obj.insert("delta".into(), message); + } + } + } + } + + serde_json::to_string(&value).ok().or(Some(text)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ChatResponseFormat, ChatToolChoice}; + + fn serialize_with(f: impl FnOnce(&mut ChatClientSettings)) -> Value { + let mut s = ChatClientSettings::default(); + f(&mut s); + s.serialize() + } + + #[test] + fn response_format_json_schema_uses_snake_case_key() { + // The native `*-json` converter (MapGuidance) reads `json_schema`, not + // `jsonSchema`; a camelCase key is silently dropped. + let v = serialize_with(|s| { + s.response_format = Some(ChatResponseFormat::JsonSchema( + "{\"type\":\"object\"}".into(), + )); + }); + let rf = &v["response_format"]; + assert_eq!(rf["type"], "json_schema"); + assert_eq!(rf["json_schema"], "{\"type\":\"object\"}"); + assert!( + rf.get("jsonSchema").is_none(), + "must not emit camelCase key" + ); + } + + #[test] + fn response_format_lark_grammar_uses_snake_case_key() { + let v = serialize_with(|s| { + s.response_format = Some(ChatResponseFormat::LarkGrammar("start: WORD+".into())); + }); + let rf = &v["response_format"]; + assert_eq!(rf["type"], "lark_grammar"); + assert_eq!(rf["lark_grammar"], "start: WORD+"); + assert!( + rf.get("larkGrammar").is_none(), + "must not emit camelCase key" + ); + } + + #[test] + fn tool_choice_simple_modes_are_plain_strings() { + // Native reads none/auto/required as JSON strings (tc.is_string()). + for (choice, expected) in [ + (ChatToolChoice::None, "none"), + (ChatToolChoice::Auto, "auto"), + (ChatToolChoice::Required, "required"), + ] { + let v = serialize_with(|s| s.tool_choice = Some(choice)); + assert_eq!(v["tool_choice"], expected); + } + } + + #[test] + fn tool_choice_function_nests_name_under_function() { + // Native reads the target as tc["function"]["name"]. + let v = serialize_with(|s| { + s.tool_choice = Some(ChatToolChoice::Function("get_weather".into())); + }); + let tc = &v["tool_choice"]; + assert_eq!(tc["type"], "function"); + assert_eq!(tc["function"]["name"], "get_weather"); + assert!( + tc.get("name").is_none(), + "name must be nested under `function`" + ); + } + + #[test] + fn foundry_metadata_carries_top_k_and_random_seed() { + let v = serialize_with(|s| { + s.top_k = Some(40); + s.random_seed = Some(7); + }); + assert_eq!(v["metadata"]["top_k"], "40"); + assert_eq!(v["metadata"]["random_seed"], "7"); + } +} diff --git a/sdk_v2/rust/src/openai/embedding_client.rs b/sdk_v2/rust/src/openai/embedding_client.rs new file mode 100644 index 000000000..71c1e01ca --- /dev/null +++ b/sdk_v2/rust/src/openai/embedding_client.rs @@ -0,0 +1,104 @@ +//! OpenAI-compatible embedding client. +#![allow(deprecated)] // this module implements the deprecated OpenAI facade + +use async_openai::types::embeddings::CreateEmbeddingResponse; +use serde_json::{json, Value}; + +use crate::detail::native::NativeModel; +use crate::detail::session::NativeSession; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; + +/// Client for OpenAI-compatible embedding generation backed by a local model. +#[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use the Session API instead \ + (`EmbeddingsSession::new(&model)`)." +)] +pub struct EmbeddingClient { + model_id: String, + model: NativeModel, +} + +impl EmbeddingClient { + pub(crate) fn new(model_id: &str, model: NativeModel) -> Self { + Self { + model_id: model_id.to_owned(), + model, + } + } + + /// Generate embeddings for a single input text. + pub async fn generate_embedding(&self, input: &str) -> Result { + Self::validate_input(input)?; + let request = self.build_request(json!(input)); + self.execute_request(request).await + } + + /// Generate embeddings for multiple input texts in a single request. + pub async fn generate_embeddings(&self, inputs: &[&str]) -> Result { + if inputs.is_empty() { + return Err(FoundryLocalError::Validation { + reason: "inputs must be a non-empty array".into(), + }); + } + for input in inputs { + Self::validate_input(input)?; + } + let request = self.build_request(json!(inputs)); + self.execute_request(request).await + } + + async fn execute_request(&self, request: Value) -> Result { + let request_json = serde_json::to_string(&request)?; + let model = self.model.clone(); + + let raw = spawn_blocking(move || { + let session = NativeSession::create(&model)?; + session.run_openai_json(&request_json) + }) + .await?; + + // Patch the response to add fields required by async_openai types + // that the server doesn't return (object on each item, usage) + let mut response_value: Value = serde_json::from_str(&raw)?; + if let Some(data) = response_value + .get_mut("data") + .and_then(|d| d.as_array_mut()) + { + for item in data { + if item.get("object").is_none() { + item.as_object_mut() + .map(|m| m.insert("object".into(), json!("embedding"))); + } + } + } + if response_value.get("usage").is_none() { + response_value.as_object_mut().map(|m| { + m.insert( + "usage".into(), + json!({"prompt_tokens": 0, "total_tokens": 0}), + ) + }); + } + + let parsed: CreateEmbeddingResponse = serde_json::from_value(response_value)?; + Ok(parsed) + } + + fn build_request(&self, input: Value) -> Value { + json!({ + "model": self.model_id, + "input": input, + }) + } + + fn validate_input(input: &str) -> Result<()> { + if input.trim().is_empty() { + return Err(FoundryLocalError::Validation { + reason: "input must be a non-empty string".into(), + }); + } + Ok(()) + } +} diff --git a/sdk_v2/rust/src/openai/json_stream.rs b/sdk_v2/rust/src/openai/json_stream.rs new file mode 100644 index 000000000..b9dca189a --- /dev/null +++ b/sdk_v2/rust/src/openai/json_stream.rs @@ -0,0 +1,49 @@ +//! Generic JSON-deserializing stream over an unbounded channel of raw strings. + +use std::marker::PhantomData; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use serde::de::DeserializeOwned; + +use crate::error::{FoundryLocalError, Result}; + +/// A stream that deserializes each received string chunk into `T`. +/// +/// Empty chunks are silently skipped. +pub struct JsonStream { + rx: tokio::sync::mpsc::UnboundedReceiver>, + _marker: PhantomData, +} + +impl JsonStream { + pub(crate) fn new(rx: tokio::sync::mpsc::UnboundedReceiver>) -> Self { + Self { + rx, + _marker: PhantomData, + } + } +} + +impl Unpin for JsonStream {} + +impl futures_core::Stream for JsonStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + match self.rx.poll_recv(cx) { + Poll::Ready(Some(Ok(chunk))) => { + if chunk.is_empty() { + continue; + } + let parsed = serde_json::from_str::(&chunk).map_err(FoundryLocalError::from); + return Poll::Ready(Some(parsed)); + } + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(None) => return Poll::Ready(None), + Poll::Pending => return Poll::Pending, + } + } + } +} diff --git a/sdk_v2/rust/src/openai/live_audio_session.rs b/sdk_v2/rust/src/openai/live_audio_session.rs new file mode 100644 index 000000000..9cdbbef01 --- /dev/null +++ b/sdk_v2/rust/src/openai/live_audio_session.rs @@ -0,0 +1,517 @@ +//! Live audio transcription streaming session. +//! +//! Provides real-time audio streaming ASR (Automatic Speech Recognition). +//! Audio data from a microphone (or other source) is pushed in as PCM chunks +//! and transcription results are returned as an async [`Stream`](futures_core::Stream). +//! +//! # Example +//! +//! ```ignore +//! let audio_client = model.create_audio_client(); +//! let mut session = audio_client.create_live_transcription_session(); +//! session.settings.sample_rate = 16000; +//! session.settings.channels = 1; +//! session.settings.language = Some("en".into()); +//! +//! session.start(None).await?; +//! +//! // Push audio from microphone callback +//! session.append(&pcm_bytes, None).await?; +//! +//! // Read results as async stream +//! use tokio_stream::StreamExt; +//! let mut stream = session.get_stream().await?; +//! while let Some(result) = stream.next().await { +//! let result = result?; +//! print!("{}", result.content[0].text); +//! } +//! +//! session.stop(None).await?; +//! ``` +#![allow(deprecated)] // this module implements the deprecated OpenAI facade + +use std::os::raw::c_int; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio_util::sync::CancellationToken; + +use crate::detail::api::Api; +use crate::detail::ffi::{flItem, flStreamingCallbackData, FOUNDRY_LOCAL_ITEM_BYTES}; +use crate::detail::items::{ + make_audio_item, read_speech_segment, read_text_item, SpeechSegmentText, +}; +use crate::detail::native::NativeModel; +use crate::detail::session::{NativeItemQueue, NativeRequest, NativeSession}; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; + +// ── Types ──────────────────────────────────────────────────────────────────── + +/// Audio format settings for a live transcription session. +/// +/// Must be configured before calling [`LiveAudioTranscriptionSession::start`]. +/// Settings are frozen once the session starts. +#[derive(Debug, Clone)] +pub struct LiveAudioTranscriptionOptions { + /// PCM sample rate in Hz. Default: 16000. + pub sample_rate: u32, + /// Number of audio channels. Default: 1 (mono). + pub channels: u32, + /// Optional BCP-47 language hint (e.g., `"en"`, `"zh"`). + pub language: Option, +} + +impl Default for LiveAudioTranscriptionOptions { + fn default() -> Self { + Self { + sample_rate: 16000, + channels: 1, + language: None, + } + } +} + +/// Internal raw deserialization target matching the native core's JSON format. +#[derive(Debug, Clone, serde::Deserialize)] +struct LiveAudioTranscriptionRaw { + #[serde(default)] + is_final: bool, + #[serde(default)] + text: String, + start_time: Option, + end_time: Option, + id: Option, +} + +/// A content part within a [`LiveAudioTranscriptionResponse`]. +/// +/// Mirrors the C# `ContentPart` shape from the OpenAI Realtime API so that +/// callers can access `result.content[0].text` or `result.content[0].transcript` +/// consistently across SDKs. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ContentPart { + /// The transcribed text. + pub text: String, + /// Same as `text` — provided for OpenAI Realtime API compatibility. + pub transcript: String, +} + +/// Transcription result from a live audio streaming session. +/// +/// Shaped to match the C# `LiveAudioTranscriptionResponse : ConversationItem` +/// so that callers access text via `result.content[0].text` or +/// `result.content[0].transcript`. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LiveAudioTranscriptionResponse { + /// Content parts — typically a single element. Access text via + /// `result.content[0].text` or `result.content[0].transcript`. + pub content: Vec, + /// Whether this is a final or partial (interim) result. + /// Nemotron models always return `true`; other models may return `false` + /// for interim hypotheses that will be replaced by a subsequent final result. + pub is_final: bool, + /// Start time offset of this segment in the audio stream (seconds). + pub start_time: Option, + /// End time offset of this segment in the audio stream (seconds). + pub end_time: Option, + /// Unique identifier for this result (if available). + pub id: Option, +} + +impl LiveAudioTranscriptionResponse { + /// Parse a transcription response from the native core's JSON format. + pub fn from_json(json: &str) -> Result { + serde_json::from_str::(json) + .map(Self::from_raw) + .map_err(FoundryLocalError::from) + } + + fn from_raw(raw: LiveAudioTranscriptionRaw) -> Self { + Self { + content: vec![ContentPart { + transcript: raw.text.clone(), + text: raw.text, + }], + is_final: raw.is_final, + start_time: raw.start_time, + end_time: raw.end_time, + id: raw.id, + } + } + + /// Build a response from a plain transcript string. + fn from_text(text: String, is_final: bool) -> Self { + Self { + content: vec![ContentPart { + transcript: text.clone(), + text, + }], + is_final, + start_time: None, + end_time: None, + id: None, + } + } + + /// Build a response from a SPEECH_SEGMENT item's content. + fn from_segment(seg: SpeechSegmentText) -> Self { + Self { + content: vec![ContentPart { + transcript: seg.text.clone(), + text: seg.text, + }], + is_final: seg.is_final, + start_time: seg.start_time_s, + end_time: seg.end_time_s, + id: None, + } + } +} + +/// Structured error response from the native core. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CoreErrorResponse { + /// Error code (e.g. `"ASR_SESSION_NOT_FOUND"`). + pub code: String, + /// Human-readable error message. + pub message: String, + /// Whether this error is transient (retryable). + #[serde(rename = "isTransient", default)] + pub is_transient: bool, +} + +impl CoreErrorResponse { + /// Attempt to parse a native error string as structured JSON. + /// Returns `None` if the error is not valid JSON or doesn't match the schema. + pub fn try_parse(error_string: &str) -> Option { + serde_json::from_str(error_string).ok() + } +} + +// ── Stream type ────────────────────────────────────────────────────────────── + +/// An async stream of [`LiveAudioTranscriptionResponse`] items. +/// +/// Returned by [`LiveAudioTranscriptionSession::get_stream`]. +/// Implements [`futures_core::Stream`]. +pub struct LiveAudioTranscriptionStream { + rx: UnboundedReceiver>, +} + +impl futures_core::Stream for LiveAudioTranscriptionStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_recv(cx) + } +} + +// ── Session state ──────────────────────────────────────────────────────────── + +#[derive(Default)] +struct SessionState { + started: bool, + stopped: bool, + queue: Option>, + output_rx: Option>>, + worker: Option>, +} + +// ── Session ────────────────────────────────────────────────────────────────── + +/// Session for real-time audio streaming ASR (Automatic Speech Recognition). +/// +/// Audio data from a microphone (or other source) is pushed in as PCM chunks +/// via [`append`](Self::append), and transcription results are returned as an +/// async [`Stream`](futures_core::Stream) via [`get_stream`](Self::get_stream). +/// +/// Created via [`AudioClient::create_live_transcription_session`](super::AudioClient::create_live_transcription_session). +/// +/// # Cancellation +/// +/// All lifecycle methods accept an optional [`CancellationToken`]. Pass `None` +/// to use the default (no cancellation). +#[deprecated( + since = "2.0.0", + note = "The OpenAI direct clients are deprecated; use the Session API instead \ + (`AudioSession::new(&model)` with streaming)." +)] +pub struct LiveAudioTranscriptionSession { + model: NativeModel, + /// Audio format settings. Must be configured before calling [`start`](Self::start). + /// Settings are frozen once the session starts. + pub settings: LiveAudioTranscriptionOptions, + state: tokio::sync::Mutex, +} + +impl LiveAudioTranscriptionSession { + pub(crate) fn new(_model_id: &str, model: NativeModel) -> Self { + Self { + model, + settings: LiveAudioTranscriptionOptions::default(), + state: tokio::sync::Mutex::new(SessionState::default()), + } + } + + /// Start a real-time audio streaming session. + /// + /// Must be called before [`append`](Self::append) or + /// [`get_stream`](Self::get_stream). Settings are frozen after this call. + pub async fn start(&self, ct: Option) -> Result<()> { + let mut state = self.state.lock().await; + + if state.started { + return Err(FoundryLocalError::Validation { + reason: "Streaming session already started. Call stop() first.".into(), + }); + } + + if let Some(token) = &ct { + if token.is_cancelled() { + return Err(FoundryLocalError::CommandExecution { + reason: "Start cancelled".into(), + }); + } + } + + let settings = self.settings.clone(); + let model = self.model.clone(); + let api = Arc::clone(&model.api); + + // Create the shared input queue up front so `append` can push into it. + let queue = Arc::new(NativeItemQueue::new(api)?); + + let (output_tx, output_rx) = + tokio::sync::mpsc::unbounded_channel::>(); + + let worker_queue = Arc::clone(&queue); + let worker = tokio::task::spawn_blocking(move || { + run_worker(model, settings, worker_queue, output_tx); + }); + + state.started = true; + state.stopped = false; + state.queue = Some(queue); + state.output_rx = Some(output_rx); + state.worker = Some(worker); + + Ok(()) + } + + /// Push a chunk of raw PCM audio data to the streaming session. + /// + /// The data is copied internally so the caller can reuse the buffer. + pub async fn append(&self, pcm_data: &[u8], ct: Option) -> Result<()> { + if let Some(token) = &ct { + if token.is_cancelled() { + return Err(FoundryLocalError::CommandExecution { + reason: "Append cancelled".into(), + }); + } + } + + let queue = { + let state = self.state.lock().await; + if !state.started || state.stopped { + return Err(FoundryLocalError::Validation { + reason: "No active streaming session. Call start() first.".into(), + }); + } + state + .queue + .clone() + .ok_or_else(|| FoundryLocalError::Internal { + reason: "Input queue not available — session may be in an invalid state".into(), + })? + }; + + let data = pcm_data.to_vec(); + spawn_blocking(move || queue.push_bytes(&data, FOUNDRY_LOCAL_ITEM_BYTES)).await + } + + /// Get the async stream of transcription results. + /// + /// Results arrive as the native ASR engine processes audio data. + /// Can only be called once per session (the receiver is moved out). + pub async fn get_stream(&self) -> Result { + let mut state = self.state.lock().await; + let rx = state + .output_rx + .take() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "No active streaming session, or stream already taken. \ + Call start() first and only call get_stream() once." + .into(), + })?; + Ok(LiveAudioTranscriptionStream { rx }) + } + + /// Signal end-of-audio and stop the streaming session. + /// + /// Any remaining buffered audio is drained to the native engine first; + /// final results are delivered through the transcription stream before it + /// completes. The native stop always completes to avoid session leaks, + /// even if the provided [`CancellationToken`] fires. + pub async fn stop(&self, _ct: Option) -> Result<()> { + let worker = { + let mut state = self.state.lock().await; + if !state.started || state.stopped { + return Ok(()); + } + state.stopped = true; + if let Some(queue) = &state.queue { + queue.mark_finished(); + } + state.worker.take() + }; + + if let Some(handle) = worker { + let _ = handle.await; + } + Ok(()) + } +} + +impl Drop for LiveAudioTranscriptionSession { + /// Best-effort cleanup if the session is dropped without calling + /// [`stop`](Self::stop): mark the input queue finished so the blocking + /// background worker drains and returns (releasing the native session), + /// rather than leaking the worker thread. We hold `&mut self`, so the inner + /// state is reachable synchronously via `get_mut` without locking. + fn drop(&mut self) { + let state = self.state.get_mut(); + if state.started && !state.stopped { + state.stopped = true; + if let Some(queue) = &state.queue { + queue.mark_finished(); + } + // Detach the worker: marking the queue finished lets its blocking + // ProcessRequest return on its own, so we don't need to (and can't) + // await it here. + state.worker.take(); + } + } +} + +/// Streaming-callback context: forwards interim transcripts to the output channel. +struct LiveCtx { + api: Arc, + tx: UnboundedSender>, +} + +unsafe extern "C" fn live_trampoline( + data: flStreamingCallbackData, + user_data: *mut std::ffi::c_void, +) -> c_int { + if user_data.is_null() { + return 0; + } + let result = catch_unwind(AssertUnwindSafe(|| { + let ctx = &*(user_data as *const LiveCtx); + let queue = data.item_queue; + if queue.is_null() { + return 0; + } + let item_api = ctx.api.item_api(); + loop { + let mut item: *mut flItem = std::ptr::null_mut(); + if !(item_api.ItemQueue_TryPop)(queue, &mut item) { + break; + } + if item.is_null() { + continue; + } + // The native audio path now streams SPEECH_SEGMENT items; older cores + // (and the OpenAI-JSON path) stream plain TEXT items. Handle both. + let response = if let Some(text) = read_text_item(&ctx.api, item) { + (!text.is_empty()).then(|| LiveAudioTranscriptionResponse::from_text(text, false)) + } else { + read_speech_segment(&ctx.api, item) + .filter(|seg| !seg.text.is_empty()) + .map(LiveAudioTranscriptionResponse::from_segment) + }; + (item_api.Item_Release)(item); + if let Some(response) = response { + if ctx.tx.send(Ok(response)).is_err() { + return 1; // receiver dropped — cancel + } + } + } + 0 + })); + result.unwrap_or(1) +} + +/// Blocking worker: builds the session/request, installs the streaming callback, +/// processes the audio queue to completion, then emits the final transcript. +fn run_worker( + model: NativeModel, + settings: LiveAudioTranscriptionOptions, + queue: Arc, + output_tx: UnboundedSender>, +) { + let api = Arc::clone(&model.api); + + let run = (|| -> Result<()> { + let session = NativeSession::create(&model)?; + + let mut ctx = Box::new(LiveCtx { + api: Arc::clone(&api), + tx: output_tx.clone(), + }); + let ctx_ptr = &mut *ctx as *mut LiveCtx as *mut std::ffi::c_void; + session.set_streaming_callback(Some(live_trampoline), ctx_ptr)?; + + let request = NativeRequest::new(Arc::clone(&api))?; + let format = make_audio_item( + &api, + &[], + Some("pcm"), + settings.sample_rate as i32, + settings.channels as i32, + )?; + request.add_item(format, true)?; + // The input queue stays owned by us (append pushes into it). + request.add_item(queue.as_item_ptr(), false)?; + + let response = session.process_request(&request); + + // Always uninstall the streaming callback before `ctx` can be dropped — + // on the error path too — so the native session never retains a dangling + // `user_data` pointer into the freed context. + let _ = session.set_streaming_callback(None, std::ptr::null_mut()); + let response = response?; + + // Aggregate the terminal transcript from the final response items. The + // native audio path returns a SPEECH_RESULT item; the OpenAI-JSON path + // (and older cores) return TEXT items. + let mut final_text = String::new(); + for i in 0..response.item_count() { + if let Some(text) = response + .item_text(i) + .or_else(|| response.item_speech_result_text(i)) + { + final_text.push_str(&text); + } + } + + drop(ctx); + + if !final_text.is_empty() { + let _ = output_tx.send(Ok(LiveAudioTranscriptionResponse::from_text( + final_text, true, + ))); + } + Ok(()) + })(); + + if let Err(e) = run { + let _ = output_tx.send(Err(e)); + } + // `queue` Arc clone and `session`/`request` drop here. + drop(queue); +} diff --git a/sdk_v2/rust/src/openai/mod.rs b/sdk_v2/rust/src/openai/mod.rs new file mode 100644 index 000000000..629f9b677 --- /dev/null +++ b/sdk_v2/rust/src/openai/mod.rs @@ -0,0 +1,19 @@ +#![allow(deprecated)] // re-exports the deprecated OpenAI facade types + +mod audio_client; +mod chat_client; +mod embedding_client; +mod json_stream; +mod live_audio_session; + +pub use self::audio_client::{ + AudioClient, AudioClientSettings, AudioTranscriptionResponse, AudioTranscriptionStream, + TranscriptionSegment, TranscriptionWord, +}; +pub use self::chat_client::{ChatClient, ChatClientSettings, ChatCompletionStream}; +pub use self::embedding_client::EmbeddingClient; +pub use self::json_stream::JsonStream; +pub use self::live_audio_session::{ + ContentPart, CoreErrorResponse, LiveAudioTranscriptionOptions, LiveAudioTranscriptionResponse, + LiveAudioTranscriptionSession, LiveAudioTranscriptionStream, +}; diff --git a/sdk_v2/rust/src/request.rs b/sdk_v2/rust/src/request.rs new file mode 100644 index 000000000..09331a832 --- /dev/null +++ b/sdk_v2/rust/src/request.rs @@ -0,0 +1,249 @@ +//! The [`Request`] value type and its [`RequestOptions`]. +//! +//! Like [`Item`], a `Request` is pure data: it owns its input items, an optional +//! borrowed streaming [`ItemQueue`], and optional [`RequestOptions`]. The native +//! `flRequest` is built transiently inside [`Session`](crate::Session) when the +//! request is processed. + +use crate::detail::ffi::{ + FOUNDRY_LOCAL_PARAM_DO_SAMPLE, FOUNDRY_LOCAL_PARAM_EARLY_STOPPING, + FOUNDRY_LOCAL_PARAM_FREQUENCY_PENALTY, FOUNDRY_LOCAL_PARAM_MAX_OUTPUT_TOKENS, + FOUNDRY_LOCAL_PARAM_PRESENCE_PENALTY, FOUNDRY_LOCAL_PARAM_SEED, + FOUNDRY_LOCAL_PARAM_TEMPERATURE, FOUNDRY_LOCAL_PARAM_TOOL_CHOICE, FOUNDRY_LOCAL_PARAM_TOP_K, + FOUNDRY_LOCAL_PARAM_TOP_P, +}; +use crate::item::Item; +use crate::item_queue::ItemQueue; + +/// How a tool-enabled request should select among available tools. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum ToolChoice { + /// Let the model decide whether and which tool to call. + #[default] + Auto, + /// Never call a tool. + None, + /// Require the model to call a tool. + Required, +} + +impl ToolChoice { + fn as_param(self) -> &'static str { + match self { + ToolChoice::Auto => "auto", + ToolChoice::None => "none", + ToolChoice::Required => "required", + } + } +} + +/// Sampling / decoding parameters applied to a request or session. +/// +/// Every field is optional; unset fields leave the model/engine default in place. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct SearchOptions { + /// Sampling temperature (higher = more random). + pub temperature: Option, + /// Nucleus-sampling probability mass in `(0, 1]`. + pub top_p: Option, + /// Top-k sampling cutoff. + pub top_k: Option, + /// Maximum number of tokens to generate. + pub max_output_tokens: Option, + /// Frequency penalty in `[-2.0, 2.0]`. + pub frequency_penalty: Option, + /// Presence penalty in `[-2.0, 2.0]`. + pub presence_penalty: Option, + /// Random seed for reproducible sampling. + pub seed: Option, + /// Stop as soon as a stop-sequence is matched. + pub early_stopping: Option, + /// Whether to sample (`false` = greedy decoding). + pub do_sample: Option, +} + +/// Options applied to a [`Request`] (or, via [`Session::set_options`], to every +/// request on a session). +/// +/// Typed [`search`](Self::search) fields and [`tool_choice`](Self::tool_choice) +/// take precedence over [`additional_options`](Self::additional_options) on key +/// collision. +/// +/// [`Session::set_options`]: crate::Session::set_options +#[derive(Debug, Clone, PartialEq, Default)] +pub struct RequestOptions { + /// Sampling / decoding parameters. + pub search: SearchOptions, + /// Tool-selection mode for tool-enabled requests. + pub tool_choice: Option, + /// Passthrough key/value options for parameters not yet typed. Applied first, + /// so typed fields win on key collision. + pub additional_options: Vec<(String, String)>, +} + +impl RequestOptions { + /// Flatten the options into ordered `(key, value)` pairs for the native layer. + /// + /// `additional_options` are emitted first so that typed `search` fields and + /// `tool_choice` override them on key collision (later writes win). + pub(crate) fn to_pairs(&self) -> Vec<(String, String)> { + let mut pairs: Vec<(String, String)> = self.additional_options.clone(); + let s = &self.search; + if let Some(v) = s.temperature { + pairs.push((FOUNDRY_LOCAL_PARAM_TEMPERATURE.to_string(), v.to_string())); + } + if let Some(v) = s.top_p { + pairs.push((FOUNDRY_LOCAL_PARAM_TOP_P.to_string(), v.to_string())); + } + if let Some(v) = s.top_k { + pairs.push((FOUNDRY_LOCAL_PARAM_TOP_K.to_string(), v.to_string())); + } + if let Some(v) = s.max_output_tokens { + pairs.push(( + FOUNDRY_LOCAL_PARAM_MAX_OUTPUT_TOKENS.to_string(), + v.to_string(), + )); + } + if let Some(v) = s.frequency_penalty { + pairs.push(( + FOUNDRY_LOCAL_PARAM_FREQUENCY_PENALTY.to_string(), + v.to_string(), + )); + } + if let Some(v) = s.presence_penalty { + pairs.push(( + FOUNDRY_LOCAL_PARAM_PRESENCE_PENALTY.to_string(), + v.to_string(), + )); + } + if let Some(v) = s.seed { + pairs.push((FOUNDRY_LOCAL_PARAM_SEED.to_string(), v.to_string())); + } + if let Some(v) = s.early_stopping { + pairs.push(( + FOUNDRY_LOCAL_PARAM_EARLY_STOPPING.to_string(), + bool_str(v).to_string(), + )); + } + if let Some(v) = s.do_sample { + pairs.push(( + FOUNDRY_LOCAL_PARAM_DO_SAMPLE.to_string(), + bool_str(v).to_string(), + )); + } + if let Some(tc) = self.tool_choice { + pairs.push(( + FOUNDRY_LOCAL_PARAM_TOOL_CHOICE.to_string(), + tc.as_param().to_string(), + )); + } + pairs + } +} + +fn bool_str(v: bool) -> &'static str { + if v { + "true" + } else { + "false" + } +} + +/// A unit of work submitted to a [`Session`](crate::Session). +/// +/// A request carries its input [`items`](Self::items), an optional streaming +/// [`input_queue`](Self::input_queue) (for incremental input such as live audio), +/// and optional [`options`](Self::options). +#[derive(Debug, Clone, Default)] +pub struct Request { + /// The input items, in order. + pub items: Vec, + /// An optional streaming input queue. When present, its items are consumed in + /// addition to [`items`](Self::items). + pub input_queue: Option, + /// Optional per-request options overriding any session options. + pub options: Option, +} + +impl Request { + /// An empty request. + pub fn new() -> Self { + Self::default() + } + + /// A request built from a list of input items. + pub fn from_items(items: impl Into>) -> Self { + Self { + items: items.into(), + input_queue: None, + options: None, + } + } + + /// Append an input item (builder-style). + pub fn with_item(mut self, item: Item) -> Self { + self.items.push(item); + self + } + + /// Attach a streaming input queue (builder-style). + pub fn with_input_queue(mut self, queue: ItemQueue) -> Self { + self.input_queue = Some(queue); + self + } + + /// Attach per-request options (builder-style). + pub fn with_options(mut self, options: RequestOptions) -> Self { + self.options = Some(options); + self + } + + /// The flattened option pairs, or an empty vector if no options are set. + pub(crate) fn option_pairs(&self) -> Vec<(String, String)> { + self.options + .as_ref() + .map(RequestOptions::to_pairs) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn typed_fields_override_additional_options() { + let opts = RequestOptions { + search: SearchOptions { + temperature: Some(0.5), + max_output_tokens: Some(128), + do_sample: Some(false), + ..Default::default() + }, + tool_choice: Some(ToolChoice::Required), + additional_options: vec![("temperature".into(), "9.9".into())], + }; + let pairs = opts.to_pairs(); + // additional_options are emitted first, typed fields after (later wins). + assert_eq!(pairs[0], ("temperature".to_string(), "9.9".to_string())); + assert!(pairs.iter().any(|(k, v)| k == "temperature" && v == "0.5")); + assert!(pairs.iter().rposition(|(k, _)| k == "temperature").unwrap() > 0); + assert!(pairs + .iter() + .any(|(k, v)| k == "max_output_tokens" && v == "128")); + assert!(pairs.iter().any(|(k, v)| k == "do_sample" && v == "false")); + assert!(pairs + .iter() + .any(|(k, v)| k == "tool_choice" && v == "required")); + } + + #[test] + fn builder_assembles_request() { + let req = Request::from_items(vec![Item::text("hi")]) + .with_item(Item::text("there")) + .with_options(RequestOptions::default()); + assert_eq!(req.items.len(), 2); + assert!(req.options.is_some()); + assert!(req.input_queue.is_none()); + } +} diff --git a/sdk_v2/rust/src/response.rs b/sdk_v2/rust/src/response.rs new file mode 100644 index 000000000..5128a264e --- /dev/null +++ b/sdk_v2/rust/src/response.rs @@ -0,0 +1,143 @@ +//! The [`Response`] value type produced by processing a [`Request`]. +//! +//! [`Request`]: crate::Request + +use crate::detail::ffi::*; +use crate::detail::session::NativeResponse; +use crate::item::Item; + +/// Why generation stopped for a [`Response`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum FinishReason { + /// No finish reason reported. + #[default] + None, + /// Generation ended due to an error. + Error, + /// The model emitted a stop condition (end-of-sequence / stop sequence). + Stop, + /// The maximum output length was reached. + Length, + /// The model requested one or more tool calls. + ToolCalls, +} + +impl FinishReason { + pub(crate) fn from_native(value: flFinishReason) -> FinishReason { + match value { + FOUNDRY_LOCAL_FINISH_ERROR => FinishReason::Error, + FOUNDRY_LOCAL_FINISH_STOP => FinishReason::Stop, + FOUNDRY_LOCAL_FINISH_LENGTH => FinishReason::Length, + FOUNDRY_LOCAL_FINISH_TOOL_CALLS => FinishReason::ToolCalls, + _ => FinishReason::None, + } + } +} + +/// Token accounting for a [`Response`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct Usage { + /// Tokens consumed by the prompt / input. + pub prompt_tokens: u32, + /// Tokens produced in the completion / output. + pub completion_tokens: u32, + /// Total tokens (`prompt_tokens + completion_tokens`). + pub total_tokens: u32, +} + +impl Usage { + /// Build usage from the native `(prompt, completion, total)` triple, + /// clamping any negative sentinel values to zero. + pub(crate) fn from_native(prompt: i64, completion: i64, total: i64) -> Usage { + Usage { + prompt_tokens: clamp_u32(prompt), + completion_tokens: clamp_u32(completion), + total_tokens: clamp_u32(total), + } + } +} + +fn clamp_u32(v: i64) -> u32 { + v.clamp(0, u32::MAX as i64) as u32 +} + +/// The result of processing a [`Request`](crate::Request): output items plus the +/// finish reason and token usage. +#[derive(Debug, Clone, PartialEq)] +pub struct Response { + /// The output items, in order. + pub items: Vec, + /// Why generation stopped. + pub finish_reason: FinishReason, + /// Token usage for the request. + pub usage: Usage, +} + +impl Response { + /// Snapshot a native response into an owned [`Response`]. + pub(crate) fn from_native(native: &NativeResponse) -> Response { + let items = native.items(); + let finish_reason = FinishReason::from_native(native.finish_reason()); + let (prompt, completion, total) = native.usage(); + Response { + items, + finish_reason, + usage: Usage::from_native(prompt, completion, total), + } + } + + /// The concatenated text of all textual output items. + /// + /// Handles both plain [`Item::Text`](crate::Item::Text) items and + /// [`Item::Message`](crate::Item::Message) items (whose text content parts are + /// concatenated); other item kinds are ignored. A convenience for chat + /// responses, where the output is typically a single assistant message. + pub fn text(&self) -> String { + let mut out = String::new(); + for item in &self.items { + match item { + Item::Text { text, .. } => out.push_str(text), + Item::Message(message) => out.push_str(&message.text()), + _ => {} + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn finish_reason_from_native() { + assert_eq!( + FinishReason::from_native(FOUNDRY_LOCAL_FINISH_STOP), + FinishReason::Stop + ); + assert_eq!(FinishReason::from_native(9999), FinishReason::None); + } + + #[test] + fn usage_clamps_negatives() { + let u = Usage::from_native(-1, 5, i64::MAX); + assert_eq!(u.prompt_tokens, 0); + assert_eq!(u.completion_tokens, 5); + assert_eq!(u.total_tokens, u32::MAX); + } + + #[test] + fn response_text_concatenates_text_and_message_items() { + let resp = Response { + items: vec![ + Item::text("a"), + Item::bytes(vec![1]), + Item::text("b"), + Item::assistant_message(vec![Item::text("c")]), + ], + finish_reason: FinishReason::Stop, + usage: Usage::default(), + }; + assert_eq!(resp.text(), "abc"); + } +} diff --git a/sdk_v2/rust/src/session.rs b/sdk_v2/rust/src/session.rs new file mode 100644 index 000000000..aefaa017e --- /dev/null +++ b/sdk_v2/rust/src/session.rs @@ -0,0 +1,439 @@ +//! The [`Session`] handle and its typed variants. +//! +//! A `Session` binds a loaded [`Model`] to a stateful native inference session. +//! Unlike the pure-data [`Item`]/[`Request`]/[`Response`] values, a session is +//! *handle-backed*: it owns a native lifetime and is shared via an `Arc`, so it +//! is cheap to clone and safe to use from multiple tasks. +//! +//! [`Item`]: crate::Item +//! [`Request`]: crate::Request +//! [`Response`]: crate::Response + +use std::ops::Deref; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use tokio::sync::mpsc::UnboundedReceiver; + +use crate::detail::api::{Api, Kvps}; +use crate::detail::model::Model; +use crate::detail::session::{run_item_streaming, NativeItemQueue, NativeRequest, NativeSession}; +use crate::detail::task::spawn_blocking; +use crate::error::{FoundryLocalError, Result}; +use crate::item::Item; +use crate::item_queue::ItemQueue; +use crate::request::{Request, RequestOptions}; +use crate::response::Response; + +/// A stateful inference session bound to a [`Model`]. +/// +/// `Session` is the low-level, modality-agnostic entry point: submit a +/// [`Request`] of [`Item`]s and receive a [`Response`] of [`Item`]s. For +/// higher-level, task-specific ergonomics use [`ChatSession`], +/// [`EmbeddingsSession`], or [`AudioSession`], which [`Deref`] to `Session`. +/// +/// Cloning a `Session` produces another handle to the same underlying native +/// session (shared conversation state). +/// +/// [`Item`]: crate::Item +#[derive(Clone)] +pub struct Session { + inner: Arc, +} + +impl Session { + /// Open a session on a loaded model. + /// + /// The model must already be loaded (see [`Model::load`]). The session + /// inherits the model's default generation parameters until overridden with + /// [`set_options`](Self::set_options) or per-request options. + /// + /// [`Model::load`]: crate::Model::load + pub async fn new(model: &Model) -> Result { + let native = model.selected_native().clone(); + let inner = spawn_blocking(move || NativeSession::create(&native)).await?; + Ok(Session { + inner: Arc::new(inner), + }) + } + + /// Process a request and return the complete response. + /// + /// Runs on a blocking worker thread; the returned future resolves when + /// generation finishes. + /// + /// # Cancellation + /// + /// The returned future is *cancel-on-drop*: if it is dropped before it + /// resolves — for example via [`tokio::time::timeout`], `tokio::select!`, or + /// aborting the task — the in-flight native request is cancelled and + /// generation stops as soon as possible rather than running to completion on + /// the detached worker. This mirrors the drop-cancels-generation behaviour of + /// [`process_streaming_request`](Self::process_streaming_request). + pub async fn process_request(&self, request: Request) -> Result { + let inner = Arc::clone(&self.inner); + // Create the native request up front (cheap) so its cancel handle can be + // shared with the drop guard before the blocking work begins. + let native = Arc::new(NativeRequest::new(Arc::clone(&inner.api))?); + let native_task = Arc::clone(&native); + let handle = tokio::task::spawn_blocking(move || { + populate_native_request(&inner.api, &native_task, &request)?; + let response = inner.process_request(&native_task)?; + Ok::(Response::from_native(&response)) + }); + + // Cancel the in-flight request if this future is dropped before the + // worker finishes; disarmed on normal completion. See `CancelGuard`. + let guard = CancelGuard::new(native); + let joined = handle.await; + guard.disarm(); + joined.map_err(|e| FoundryLocalError::Internal { + reason: format!("blocking task join error: {e}"), + })? + } + + /// Process a request, streaming each output [`Item`](crate::Item) as it is + /// produced. + /// + /// The returned [`ItemStream`] yields items until generation completes; + /// dropping it cancels generation. + pub fn process_streaming_request(&self, request: Request) -> ItemStream { + let Request { + items, + input_queue, + options, + } = request; + let option_pairs = options + .as_ref() + .map(RequestOptions::to_pairs) + .unwrap_or_default(); + let input_queue = input_queue.map(ItemQueue::into_native); + let rx = run_item_streaming(Arc::clone(&self.inner), items, input_queue, option_pairs); + ItemStream { rx } + } + + /// Apply session-scoped options that persist across subsequent requests. + pub async fn set_options(&self, options: RequestOptions) -> Result<()> { + let inner = Arc::clone(&self.inner); + spawn_blocking(move || { + let pairs = options.to_pairs(); + let kvps = Kvps::from_pairs(Arc::clone(&inner.api), pairs)?; + inner.set_options(kvps.as_ptr()) + }) + .await + } + + /// Create an [`ItemQueue`] for streaming incremental input into a request. + /// + /// Attach the returned queue to a [`Request`] via + /// [`Request::with_input_queue`], then push items as they become available. + pub fn create_input_queue(&self) -> Result { + let native = NativeItemQueue::new(Arc::clone(&self.inner.api))?; + Ok(ItemQueue::from_native(Arc::new(native))) + } +} + +impl std::fmt::Debug for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Session").finish_non_exhaustive() + } +} + +/// Populate a freshly-created native request from a pure-data [`Request`]. +/// +/// The native layer copies every buffer it needs, so the owned `Request` (and +/// its items) may be dropped as soon as processing returns. The `NativeRequest` +/// is created by the caller (rather than here) so a cancel handle can be shared +/// with the drop guard before the blocking population/processing begins. +fn populate_native_request( + api: &Arc, + native: &NativeRequest, + request: &Request, +) -> Result<()> { + for item in &request.items { + native.add_item_value(item)?; + } + if let Some(queue) = &request.input_queue { + native.add_input_queue(queue.native())?; + } + let pairs = request.option_pairs(); + if !pairs.is_empty() { + let kvps = Kvps::from_pairs(Arc::clone(api), pairs)?; + native.set_options(kvps.as_ptr())?; + } + Ok(()) +} + +/// Cancels an in-flight native request if dropped while still armed. +/// +/// [`Session::process_request`] holds one of these across the `.await` on its +/// blocking worker. If the caller drops that future first (via +/// [`tokio::time::timeout`], `tokio::select!`, task abort, …) the guard fires +/// `Request_Cancel` so generation stops promptly instead of running to +/// completion on the detached worker. Disarmed once the worker completes +/// normally. +struct CancelGuard { + native: Arc, + armed: bool, +} + +impl CancelGuard { + fn new(native: Arc) -> Self { + Self { + native, + armed: true, + } + } + + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for CancelGuard { + fn drop(&mut self) { + if self.armed { + self.native.cancel(); + } + } +} + +/// An asynchronous stream of output [`Item`](crate::Item)s from +/// [`Session::process_streaming_request`]. +/// +/// Implements [`futures_core::Stream`]; use with the `futures`/`tokio-stream` +/// combinators (e.g. `while let Some(item) = stream.next().await`). +pub struct ItemStream { + rx: UnboundedReceiver>, +} + +impl Unpin for ItemStream {} + +impl futures_core::Stream for ItemStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_recv(cx) + } +} + +/// A tool the model may call, registered on a [`ChatSession`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ToolDefinition { + /// The tool's unique name. + pub name: String, + /// An optional human/model-readable description of what the tool does. + pub description: Option, + /// A JSON Schema string describing the tool's parameters. + pub json_schema: String, +} + +impl ToolDefinition { + /// A tool definition with a name and JSON-schema parameter description. + pub fn new(name: impl Into, json_schema: impl Into) -> Self { + Self { + name: name.into(), + description: None, + json_schema: json_schema.into(), + } + } + + /// Attach a description (builder-style). + pub fn with_description(mut self, description: impl Into) -> Self { + self.description = Some(description.into()); + self + } +} + +/// A chat-oriented [`Session`] with tool registration and turn management. +/// +/// Dereferences to [`Session`], so all base methods +/// ([`process_request`](Session::process_request), +/// [`process_streaming_request`](Session::process_streaming_request), …) are +/// available directly. +#[derive(Clone)] +pub struct ChatSession { + session: Session, +} + +impl ChatSession { + /// Open a chat session on a loaded model. + pub async fn new(model: &Model) -> Result { + Ok(ChatSession { + session: Session::new(model).await?, + }) + } + + /// Register a [`ToolDefinition`] for the lifetime of the session. + pub async fn add_tool_definition(&self, definition: ToolDefinition) -> Result<()> { + let inner = Arc::clone(&self.session.inner); + spawn_blocking(move || { + inner.add_tool_definition( + &definition.name, + definition.description.as_deref(), + &definition.json_schema, + ) + }) + .await + } + + /// Remove a previously-registered tool by name. Returns whether one was removed. + pub async fn remove_tool_definition(&self, name: impl Into) -> Result { + let inner = Arc::clone(&self.session.inner); + let name = name.into(); + spawn_blocking(move || inner.remove_tool_definition(&name)).await + } + + /// The number of completed conversation turns. + pub fn turn_count(&self) -> usize { + self.session.inner.turn_count() + } + + /// Rewind the last `count` turns, dropping their messages and replies. + pub async fn undo_turns(&self, count: usize) -> Result<()> { + let inner = Arc::clone(&self.session.inner); + spawn_blocking(move || inner.undo_turns(count)).await + } + + /// Consume this handle, yielding the underlying base [`Session`]. + pub fn into_session(self) -> Session { + self.session + } +} + +impl Deref for ChatSession { + type Target = Session; + + fn deref(&self) -> &Session { + &self.session + } +} + +/// An embeddings-oriented [`Session`] producing dense vectors for text input. +/// +/// Dereferences to [`Session`]. +#[derive(Clone)] +pub struct EmbeddingsSession { + session: Session, +} + +impl EmbeddingsSession { + /// Open an embeddings session on a loaded model. + pub async fn new(model: &Model) -> Result { + Ok(EmbeddingsSession { + session: Session::new(model).await?, + }) + } + + /// Embed a single text input, returning its dense vector. + pub async fn embed(&self, input: impl Into) -> Result> { + let vectors = self.embed_batch(vec![input.into()]).await?; + vectors + .into_iter() + .next() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "embeddings response contained no vectors".to_string(), + }) + } + + /// Embed a batch of text inputs, returning one dense vector per input (in + /// order). + pub async fn embed_batch(&self, inputs: Vec) -> Result>> { + let items: Vec = inputs.iter().map(|s| Item::text(s.as_str())).collect(); + let request = Request::from_items(items); + let response = self.session.process_request(request).await?; + + if response.items.len() != inputs.len() { + return Err(FoundryLocalError::Validation { + reason: format!( + "embeddings response returned {} vectors for {} inputs", + response.items.len(), + inputs.len() + ), + }); + } + + let mut vectors = Vec::with_capacity(response.items.len()); + for item in &response.items { + let tensor = item + .as_tensor() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "embeddings response item was not a tensor".to_string(), + })?; + let floats = tensor + .as_f32() + .ok_or_else(|| FoundryLocalError::Validation { + reason: "embeddings tensor was not float data".to_string(), + })?; + vectors.push(floats); + } + Ok(vectors) + } + + /// Consume this handle, yielding the underlying base [`Session`]. + pub fn into_session(self) -> Session { + self.session + } +} + +impl Deref for EmbeddingsSession { + type Target = Session; + + fn deref(&self) -> &Session { + &self.session + } +} + +/// An audio-oriented [`Session`] for speech tasks (e.g. transcription). +/// +/// Dereferences to [`Session`]. Submit [`Item::audio_data`](crate::Item::audio_data) +/// / [`Item::audio_uri`](crate::Item::audio_uri) input and read +/// [`Item::SpeechResult`](crate::Item::SpeechResult) output, or use +/// [`transcribe`](Self::transcribe) for the common case. +#[derive(Clone)] +pub struct AudioSession { + session: Session, +} + +impl AudioSession { + /// Open an audio session on a loaded model. + pub async fn new(model: &Model) -> Result { + Ok(AudioSession { + session: Session::new(model).await?, + }) + } + + /// Transcribe a single audio item, returning the recognized text. + /// + /// A convenience over [`process_request`](Session::process_request): submits + /// `audio` and concatenates the text of every returned speech result. + pub async fn transcribe(&self, audio: Item) -> Result { + let response = self + .session + .process_request(Request::from_items(vec![audio])) + .await?; + let mut text = String::new(); + for item in &response.items { + if let Some(result) = item.as_speech_result() { + text.push_str(&result.text); + } else if let Some(t) = item.as_text() { + text.push_str(t); + } + } + Ok(text) + } + + /// Consume this handle, yielding the underlying base [`Session`]. + pub fn into_session(self) -> Session { + self.session + } +} + +impl Deref for AudioSession { + type Target = Session; + + fn deref(&self) -> &Session { + &self.session + } +} diff --git a/sdk_v2/rust/src/types.rs b/sdk_v2/rust/src/types.rs new file mode 100644 index 000000000..f39109adf --- /dev/null +++ b/sdk_v2/rust/src/types.rs @@ -0,0 +1,151 @@ +use serde::{Deserialize, Serialize}; + +/// Hardware device type for model execution. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeviceType { + Invalid, + #[default] + CPU, + GPU, + NPU, +} + +/// Prompt template describing how messages are formatted for the model. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PromptTemplate { + #[serde(skip_serializing_if = "Option::is_none")] + pub system: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub assistant: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt: Option, +} + +/// Runtime information for a model (device type and execution provider). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Runtime { + pub device_type: DeviceType, + pub execution_provider: String, +} + +/// A single parameter key-value pair used in model settings. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Parameter { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, +} + +/// Model-level settings containing a list of parameters. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelSettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub parameters: Option>, +} + +/// Full metadata for a model variant as returned by the catalog. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelInfo { + pub id: String, + pub name: String, + pub version: u64, + pub alias: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, + pub provider_type: String, + pub uri: String, + pub model_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_template: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub publisher: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub model_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub license: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub license_description: Option, + pub cached: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub task: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_size_mb: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub supports_tool_calling: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub min_fl_version: Option, + #[serde(default)] + pub created_at_unix: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub context_length: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_modalities: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_modalities: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub capabilities: Option, +} + +/// Desired response format for chat completions. +/// +/// Extends the standard OpenAI formats with the Foundry-specific +/// `LarkGrammar` variant. +#[derive(Debug, Clone)] +pub enum ChatResponseFormat { + /// Plain text output (default). + Text, + /// JSON output (unstructured). + JsonObject, + /// JSON output constrained by a schema string. + JsonSchema(String), + /// Output constrained by a Lark grammar (Foundry extension). + LarkGrammar(String), +} + +/// Tool choice configuration for chat completions. +#[derive(Debug, Clone)] +pub enum ChatToolChoice { + /// Model will not call any tool. + None, + /// Model decides whether to call a tool. + Auto, + /// Model must call at least one tool. + Required, + /// Model must call the named function. + Function(String), +} + +/// Information about an available execution provider. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct EpInfo { + /// The name of the execution provider. + pub name: String, + /// Whether this EP is currently registered and ready for use. + pub is_registered: bool, +} + +/// Result of a download-and-register execution-provider operation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct EpDownloadResult { + /// Whether all requested EPs were successfully registered. + pub success: bool, + /// Human-readable status message. + pub status: String, + /// Names of EPs that were successfully registered. + pub registered_eps: Vec, + /// Names of EPs that failed to register. + pub failed_eps: Vec, +} diff --git a/sdk_v2/rust/tests/integration/audio_client_test.rs b/sdk_v2/rust/tests/integration/audio_client_test.rs new file mode 100644 index 000000000..47cef9d01 --- /dev/null +++ b/sdk_v2/rust/tests/integration/audio_client_test.rs @@ -0,0 +1,130 @@ +use super::common; +use foundry_local_sdk::openai::AudioClient; +use std::sync::Arc; +use tokio_stream::StreamExt; + +async fn setup_audio_client() -> (AudioClient, Arc) { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let model = catalog + .get_model(common::WHISPER_MODEL_ALIAS) + .await + .expect("get_model(whisper-tiny) failed"); + model.load().await.expect("model.load() failed"); + (model.create_audio_client(), model) +} + +fn audio_file() -> String { + common::get_audio_file_path().to_string_lossy().into_owned() +} + +#[tokio::test] +async fn should_transcribe_audio_without_streaming() { + let (client, model) = setup_audio_client().await; + let client = client.language("en").temperature(0.0); + let response = client + .transcribe(&audio_file()) + .await + .expect("transcribe failed"); + + assert!( + response.text.contains(common::EXPECTED_TRANSCRIPTION_TEXT), + "Transcription should contain expected text, got: {}", + response.text + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_transcribe_audio_without_streaming_with_temperature() { + let (client, model) = setup_audio_client().await; + let client = client.language("en").temperature(0.5); + + let response = client + .transcribe(&audio_file()) + .await + .expect("transcribe with temperature failed"); + + assert!( + response.text.contains(common::EXPECTED_TRANSCRIPTION_TEXT), + "Transcription should contain expected text, got: {}", + response.text + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_transcribe_audio_with_streaming() { + let (client, model) = setup_audio_client().await; + let client = client.language("en").temperature(0.0); + let mut full_text = String::new(); + + let mut stream = client + .transcribe_streaming(&audio_file()) + .await + .expect("transcribe_streaming setup failed"); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + full_text.push_str(&chunk.text); + } + + println!("Streamed transcription: {full_text}"); + + assert!( + full_text.contains(common::EXPECTED_TRANSCRIPTION_TEXT), + "Streamed transcription should contain expected text, got: {full_text}" + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_transcribe_audio_with_streaming_with_temperature() { + let (client, model) = setup_audio_client().await; + let client = client.language("en").temperature(0.5); + + let mut full_text = String::new(); + + let mut stream = client + .transcribe_streaming(&audio_file()) + .await + .expect("transcribe_streaming with temperature setup failed"); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + full_text.push_str(&chunk.text); + } + + println!("Streamed transcription: {full_text}"); + + assert!( + full_text.contains(common::EXPECTED_TRANSCRIPTION_TEXT), + "Streamed transcription should contain expected text, got: {full_text}" + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_throw_when_transcribing_with_empty_audio_file_path() { + let (client, model) = setup_audio_client().await; + let result = client.transcribe("").await; + assert!(result.is_err(), "Expected error for empty audio file path"); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_throw_when_transcribing_streaming_with_empty_audio_file_path() { + let (client, model) = setup_audio_client().await; + let result = client.transcribe_streaming("").await; + assert!( + result.is_err(), + "Expected error for empty audio file path in streaming" + ); + + model.unload().await.expect("model.unload() failed"); +} diff --git a/sdk_v2/rust/tests/integration/catalog_test.rs b/sdk_v2/rust/tests/integration/catalog_test.rs new file mode 100644 index 000000000..d418c7a73 --- /dev/null +++ b/sdk_v2/rust/tests/integration/catalog_test.rs @@ -0,0 +1,106 @@ +use super::common; +use foundry_local_sdk::Catalog; + +fn catalog() -> &'static Catalog { + common::get_test_manager().catalog() +} + +#[test] +fn should_initialize_with_catalog_name() { + let cat = catalog(); + let name = cat.name(); + assert!(!name.is_empty(), "Catalog name must not be empty"); +} + +#[tokio::test] +async fn should_list_models() { + let cat = catalog(); + let models = cat.get_models().await.expect("get_models failed"); + + assert!( + !models.is_empty(), + "Expected at least one model in the catalog" + ); + + let found = models.iter().any(|m| m.alias() == common::TEST_MODEL_ALIAS); + assert!( + found, + "Test model '{}' not found in catalog", + common::TEST_MODEL_ALIAS + ); +} + +#[tokio::test] +async fn should_get_model_by_alias() { + let cat = catalog(); + let model = cat + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + assert_eq!(model.alias(), common::TEST_MODEL_ALIAS); +} + +#[tokio::test] +async fn should_throw_when_getting_model_with_empty_alias() { + let cat = catalog(); + let result = cat.get_model("").await; + assert!(result.is_err(), "Expected error for empty alias"); + + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("Model alias must be a non-empty string"), + "Unexpected error message: {err_msg}" + ); +} + +#[tokio::test] +async fn should_throw_when_getting_model_with_unknown_alias() { + let cat = catalog(); + let result = cat.get_model("unknown-nonexistent-model-alias").await; + assert!(result.is_err(), "Expected error for unknown alias"); + + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("Unknown model alias"), + "Error should mention unknown alias: {err_msg}" + ); + assert!( + err_msg.contains("Available"), + "Error should list available models: {err_msg}" + ); +} + +#[tokio::test] +async fn should_get_cached_models() { + let cat = catalog(); + let cached = cat + .get_cached_models() + .await + .expect("get_cached_models failed"); + + assert!(!cached.is_empty(), "Expected at least one cached model"); + + let found = cached.iter().any(|m| m.alias() == common::TEST_MODEL_ALIAS); + assert!( + found, + "Test model '{}' should be in the cached models list", + common::TEST_MODEL_ALIAS + ); +} + +#[tokio::test] +async fn should_throw_when_getting_model_variant_with_empty_id() { + let cat = catalog(); + let result = cat.get_model_variant("").await; + assert!(result.is_err(), "Expected error for empty variant ID"); +} + +#[tokio::test] +async fn should_throw_when_getting_model_variant_with_unknown_id() { + let cat = catalog(); + let result = cat + .get_model_variant("unknown-nonexistent-variant-id") + .await; + assert!(result.is_err(), "Expected error for unknown variant ID"); +} diff --git a/sdk_v2/rust/tests/integration/chat_client_test.rs b/sdk_v2/rust/tests/integration/chat_client_test.rs new file mode 100644 index 000000000..bd4a3a20c --- /dev/null +++ b/sdk_v2/rust/tests/integration/chat_client_test.rs @@ -0,0 +1,332 @@ +use super::common; +use foundry_local_sdk::openai::ChatClient; +use foundry_local_sdk::{ + ChatCompletionMessageToolCalls, ChatCompletionRequestMessage, + ChatCompletionRequestSystemMessage, ChatCompletionRequestToolMessage, + ChatCompletionRequestUserMessage, ChatToolChoice, +}; +use serde_json::json; +use std::sync::Arc; +use tokio_stream::StreamExt; + +async fn setup_chat_client() -> (ChatClient, Arc) { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let model = catalog + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + model.load().await.expect("model.load() failed"); + + let client = model.create_chat_client().max_tokens(500).temperature(0.0); + (client, model) +} + +fn user_message(content: &str) -> ChatCompletionRequestMessage { + ChatCompletionRequestUserMessage::from(content).into() +} + +fn system_message(content: &str) -> ChatCompletionRequestMessage { + ChatCompletionRequestSystemMessage::from(content).into() +} + +fn assistant_message(content: &str) -> ChatCompletionRequestMessage { + serde_json::from_value(json!({ "role": "assistant", "content": content })) + .expect("failed to construct assistant message") +} + +#[tokio::test] +async fn should_perform_chat_completion() { + let (client, model) = setup_chat_client().await; + let messages = vec![ + system_message("You are a helpful math assistant. Respond with just the answer."), + user_message("What is 7*6?"), + ]; + + let response = client + .complete_chat(&messages, None) + .await + .expect("complete_chat failed"); + let content = response + .choices + .first() + .and_then(|c| c.message.content.as_deref()) + .unwrap_or(""); + println!("Response: {content}"); + + assert!( + content.contains("42"), + "Expected response to contain '42', got: {content}" + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_perform_streaming_chat_completion() { + let (client, model) = setup_chat_client().await; + let mut messages = vec![ + system_message("You are a helpful math assistant. Respond with just the answer."), + user_message("What is 7*6?"), + ]; + + let mut first_result = String::new(); + let mut stream = client + .complete_streaming_chat(&messages, None) + .await + .expect("streaming chat (first turn) setup failed"); + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + if let Some(choice) = chunk.choices.first() { + if let Some(ref content) = choice.delta.content { + first_result.push_str(content); + } + } + } + println!("First turn: {first_result}"); + + assert!( + first_result.contains("42"), + "First turn should contain '42', got: {first_result}" + ); + + messages.push(assistant_message(&first_result)); + messages.push(user_message("Now add 25 to that result.")); + + let mut second_result = String::new(); + let mut stream = client + .complete_streaming_chat(&messages, None) + .await + .expect("streaming chat (follow-up) setup failed"); + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + if let Some(choice) = chunk.choices.first() { + if let Some(ref content) = choice.delta.content { + second_result.push_str(content); + } + } + } + println!("Follow-up: {second_result}"); + + assert!( + second_result.contains("67"), + "Follow-up should contain '67', got: {second_result}" + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_throw_when_completing_chat_with_empty_messages() { + let (client, model) = setup_chat_client().await; + let messages: Vec = vec![]; + + let result = client.complete_chat(&messages, None).await; + assert!(result.is_err(), "Expected error for empty messages"); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_throw_when_completing_streaming_chat_with_empty_messages() { + let (client, model) = setup_chat_client().await; + let messages: Vec = vec![]; + + let result = client.complete_streaming_chat(&messages, None).await; + assert!( + result.is_err(), + "Expected error for empty messages in streaming" + ); + + model.unload().await.expect("model.unload() failed"); +} + +// Note: The "invalid callback" test was removed because it was an exact +// duplicate of should_throw_when_completing_streaming_chat_with_empty_messages. + +#[tokio::test] +async fn should_perform_tool_calling_chat_completion_non_streaming() { + let (client, model) = setup_chat_client().await; + let client = client.tool_choice(ChatToolChoice::Required); + + let tools = vec![common::get_multiply_tool()]; + let mut messages = vec![ + system_message("You are a math assistant. Use the multiply tool to answer."), + user_message("What is 6 times 7?"), + ]; + + let response = client + .complete_chat(&messages, Some(&tools)) + .await + .expect("complete_chat with tools failed"); + + let choice = response + .choices + .first() + .expect("Expected at least one choice"); + let tool_calls = choice + .message + .tool_calls + .as_ref() + .expect("Expected tool_calls"); + assert!( + !tool_calls.is_empty(), + "Expected at least one tool call in the response" + ); + + let tool_call = match &tool_calls[0] { + ChatCompletionMessageToolCalls::Function(tc) => tc, + _ => panic!("Expected a function tool call"), + }; + assert_eq!( + tool_call.function.name, "multiply", + "Expected tool call to 'multiply'" + ); + + let args: serde_json::Value = serde_json::from_str(&tool_call.function.arguments) + .expect("Failed to parse tool call arguments"); + let a = args["a"].as_f64().unwrap_or(0.0); + let b = args["b"].as_f64().unwrap_or(0.0); + let product = (a * b) as i64; + + let tool_call_id = &tool_call.id; + let assistant_msg: ChatCompletionRequestMessage = serde_json::from_value(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": tool_call_id, + "type": "function", + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + } + }] + })) + .expect("failed to construct assistant message"); + messages.push(assistant_msg); + messages.push( + ChatCompletionRequestToolMessage { + content: product.to_string().into(), + tool_call_id: tool_call_id.clone(), + } + .into(), + ); + + let client = client.tool_choice(ChatToolChoice::Auto); + + let final_response = client + .complete_chat(&messages, Some(&tools)) + .await + .expect("follow-up complete_chat with tools failed"); + let content = final_response + .choices + .first() + .and_then(|c| c.message.content.as_deref()) + .unwrap_or(""); + + println!("Tool call result: {content}"); + + assert!( + content.contains("42"), + "Final answer should contain '42', got: {content}" + ); + + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_perform_tool_calling_chat_completion_streaming() { + let (client, model) = setup_chat_client().await; + let client = client.tool_choice(ChatToolChoice::Required); + + let tools = vec![common::get_multiply_tool()]; + let mut messages = vec![ + system_message("You are a math assistant. Use the multiply tool to answer."), + user_message("What is 6 times 7?"), + ]; + + let mut tool_call_name = String::new(); + let mut tool_call_args = String::new(); + let mut tool_call_id = String::new(); + + let mut stream = client + .complete_streaming_chat(&messages, Some(&tools)) + .await + .expect("streaming tool call setup failed"); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + if let Some(choice) = chunk.choices.first() { + if let Some(ref tool_calls) = choice.delta.tool_calls { + for call in tool_calls { + if let Some(ref func) = call.function { + if let Some(ref name) = func.name { + tool_call_name.push_str(name); + } + if let Some(ref args) = func.arguments { + tool_call_args.push_str(args); + } + } + if let Some(ref id) = call.id { + tool_call_id = id.clone(); + } + } + } + } + } + assert_eq!( + tool_call_name, "multiply", + "Expected streamed tool call to 'multiply'" + ); + + let args: serde_json::Value = + serde_json::from_str(&tool_call_args).unwrap_or_else(|_| json!({})); + let a = args["a"].as_f64().unwrap_or(0.0); + let b = args["b"].as_f64().unwrap_or(0.0); + let product = (a * b) as i64; + + let assistant_msg: ChatCompletionRequestMessage = serde_json::from_value(json!({ + "role": "assistant", + "tool_calls": [{ + "id": tool_call_id, + "type": "function", + "function": { + "name": tool_call_name, + "arguments": tool_call_args + } + }] + })) + .expect("failed to construct assistant message"); + messages.push(assistant_msg); + messages.push( + ChatCompletionRequestToolMessage { + content: product.to_string().into(), + tool_call_id: tool_call_id.clone(), + } + .into(), + ); + + let client = client.tool_choice(ChatToolChoice::Auto); + + let mut final_result = String::new(); + let mut stream = client + .complete_streaming_chat(&messages, Some(&tools)) + .await + .expect("streaming follow-up setup failed"); + while let Some(chunk) = stream.next().await { + let chunk = chunk.expect("stream chunk error"); + if let Some(choice) = chunk.choices.first() { + if let Some(ref content) = choice.delta.content { + final_result.push_str(content); + } + } + } + println!("Streamed tool call result: {final_result}"); + + assert!( + final_result.contains("42"), + "Streamed final answer should contain '42', got: {final_result}" + ); + + model.unload().await.expect("model.unload() failed"); +} diff --git a/sdk_v2/rust/tests/integration/common/mod.rs b/sdk_v2/rust/tests/integration/common/mod.rs new file mode 100644 index 000000000..4b96daf3f --- /dev/null +++ b/sdk_v2/rust/tests/integration/common/mod.rs @@ -0,0 +1,136 @@ +//! Shared test utilities and configuration for Foundry Local SDK integration tests. +//! +//! Mirrors `testUtils.ts` from the JavaScript SDK test suite. + +#![allow(dead_code)] + +use std::path::PathBuf; +use std::sync::{Arc, OnceLock}; + +use foundry_local_sdk::{FoundryLocalConfig, FoundryLocalManager, LogLevel}; + +/// Default model alias used for chat-completion integration tests. +pub const TEST_MODEL_ALIAS: &str = "qwen2.5-0.5b"; + +/// Default model alias used for audio-transcription integration tests. +pub const WHISPER_MODEL_ALIAS: &str = "whisper-tiny"; + +/// Default model alias used for embedding integration tests. +pub const EMBEDDING_MODEL_ALIAS: &str = "qwen3-embedding-0.6b"; + +/// Expected transcription text fragment for the shared audio test file. +pub const EXPECTED_TRANSCRIPTION_TEXT: &str = + " And lots of times you need to give people more than one link at a time"; + +// ── Environment helpers ────────────────────────────────────────────────────── + +/// Returns `true` when the tests are running inside a CI environment +/// (Azure DevOps or GitHub Actions). +pub fn is_running_in_ci() -> bool { + let azure_devops = std::env::var("TF_BUILD").unwrap_or_else(|_| "false".into()); + let github_actions = std::env::var("GITHUB_ACTIONS").unwrap_or_else(|_| "false".into()); + azure_devops.eq_ignore_ascii_case("true") || github_actions.eq_ignore_ascii_case("true") +} + +/// Walk upward from `CARGO_MANIFEST_DIR` until a `.git` directory is found. +pub fn get_git_repo_root() -> PathBuf { + let mut current = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + loop { + if current.join(".git").exists() { + return current; + } + if !current.pop() { + panic!( + "Could not locate git repo root starting from {}", + env!("CARGO_MANIFEST_DIR") + ); + } + } +} + +/// Path to the shared test-data directory. +/// Uses FOUNDRY_TEST_DATA_DIR env var if set (CI), otherwise falls back +/// to looking for test-data-shared as a sibling of the repo root. +pub fn get_test_data_shared_path() -> PathBuf { + if let Ok(env_path) = std::env::var("FOUNDRY_TEST_DATA_DIR") { + let p = PathBuf::from(&env_path); + if p.is_dir() { + return p; + } + } + let repo_root = get_git_repo_root(); + repo_root + .parent() + .expect("repo root has no parent") + .join("test-data-shared") +} + +/// Path to the shared audio test file used by audio-client tests. +pub fn get_audio_file_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("testdata") + .join("Recording.mp3") +} + +// ── Test configuration ─────────────────────────────────────────────────────── + +/// Build a [`FoundryLocalConfig`] suitable for integration tests. +/// +/// * `modelCacheDir` → `/../test-data-shared` +/// * `logsDir` → `/sdk_v2/rust/logs` +/// * `logLevel` → `Warn` +pub fn test_config() -> FoundryLocalConfig { + let repo_root = get_git_repo_root(); + let logs_dir = repo_root.join("sdk_v2").join("rust").join("logs"); + + FoundryLocalConfig::new("FoundryLocalTest") + .model_cache_dir(get_test_data_shared_path().to_string_lossy().into_owned()) + .logs_dir(logs_dir.to_string_lossy().into_owned()) + .log_level(LogLevel::Warn) +} + +/// Create (or return the cached) [`FoundryLocalManager`] for tests. +/// +/// Holds a process-lifetime strong handle so the shared instance survives for +/// the whole test binary (avoiding repeated native init), and hands out a +/// `'static` borrow into it. +/// +/// Panics if creation fails so that test set-up failures are immediately +/// visible. +pub fn get_test_manager() -> &'static FoundryLocalManager { + static TEST_MANAGER: OnceLock> = OnceLock::new(); + TEST_MANAGER.get_or_init(|| { + FoundryLocalManager::create(test_config()).expect("Failed to create FoundryLocalManager") + }) +} + +// ── Tool definitions ───────────────────────────────────────────────────────── + +/// Returns a tool definition for a simple "multiply" function. +/// +/// Used by tool-calling chat-completion tests. +pub fn get_multiply_tool() -> foundry_local_sdk::ChatCompletionTools { + serde_json::from_value(serde_json::json!({ + "type": "function", + "function": { + "name": "multiply", + "description": "Multiply two numbers together", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number", + "description": "The first number" + }, + "b": { + "type": "number", + "description": "The second number" + } + }, + "required": ["a", "b"] + } + } + })) + .expect("Failed to parse multiply tool definition") +} diff --git a/sdk_v2/rust/tests/integration/embedding_client_test.rs b/sdk_v2/rust/tests/integration/embedding_client_test.rs new file mode 100644 index 000000000..0f577329a --- /dev/null +++ b/sdk_v2/rust/tests/integration/embedding_client_test.rs @@ -0,0 +1,223 @@ +//! Integration tests for EmbeddingClient. + +use std::sync::Arc; + +use foundry_local_sdk::openai::EmbeddingClient; +use foundry_local_sdk::Model; + +use crate::common; + +async fn setup_embedding_client() -> (EmbeddingClient, Arc) { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + + let model = catalog + .get_model(common::EMBEDDING_MODEL_ALIAS) + .await + .expect("embedding model should exist in catalog"); + + model.load().await.expect("model should load successfully"); + + let client = model.create_embedding_client(); + (client, model) +} + +#[tokio::test] +async fn should_generate_embedding() { + let (client, model) = setup_embedding_client().await; + + let response = client + .generate_embedding("The quick brown fox jumps over the lazy dog") + .await + .expect("embedding should succeed"); + + assert_eq!(response.data.len(), 1); + assert_eq!(response.data[0].index, 0); + assert_eq!(response.data[0].embedding.len(), 1024); + + println!("Embedding dimension: {}", response.data[0].embedding.len()); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_generate_normalized_embedding() { + let (client, model) = setup_embedding_client().await; + + let inputs = [ + "The quick brown fox jumps over the lazy dog", + "Machine learning is a subset of artificial intelligence", + "The capital of France is Paris", + ]; + + for input in &inputs { + let response = client + .generate_embedding(input) + .await + .expect("embedding should succeed"); + + let embedding = &response.data[0].embedding; + assert_eq!(embedding.len(), 1024); + + // Verify L2 norm is approximately 1.0 + let norm: f32 = embedding.iter().map(|v| v * v).sum::().sqrt(); + assert!( + (0.99_f32..=1.01_f32).contains(&norm), + "L2 norm {norm} not approximately 1.0" + ); + + for val in embedding { + assert!( + (-1.0_f32..=1.0_f32).contains(val), + "value {val} outside [-1, 1]" + ); + } + } + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_produce_different_embeddings_for_different_inputs() { + let (client, model) = setup_embedding_client().await; + + let response1 = client + .generate_embedding("The quick brown fox") + .await + .expect("embedding should succeed"); + + let response2 = client + .generate_embedding("The capital of France is Paris") + .await + .expect("embedding should succeed"); + + let emb1 = &response1.data[0].embedding; + let emb2 = &response2.data[0].embedding; + + assert_eq!(emb1.len(), emb2.len()); + + // Cosine similarity should not be 1.0 + let dot: f32 = emb1.iter().zip(emb2.iter()).map(|(a, b)| a * b).sum(); + let norm1: f32 = emb1.iter().map(|v| v * v).sum::().sqrt(); + let norm2: f32 = emb2.iter().map(|v| v * v).sum::().sqrt(); + let cosine_similarity = dot / (norm1 * norm2); + assert!( + cosine_similarity < 0.99_f32, + "cosine similarity {cosine_similarity} should be < 0.99" + ); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_produce_same_embedding_for_same_input() { + let (client, model) = setup_embedding_client().await; + + let response1 = client + .generate_embedding("Deterministic embedding test") + .await + .expect("embedding should succeed"); + + let response2 = client + .generate_embedding("Deterministic embedding test") + .await + .expect("embedding should succeed"); + + let emb1 = &response1.data[0].embedding; + let emb2 = &response2.data[0].embedding; + + for (i, (a, b)) in emb1.iter().zip(emb2.iter()).enumerate() { + assert_eq!(a, b, "mismatch at index {i}"); + } + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_throw_for_empty_input() { + let (client, model) = setup_embedding_client().await; + + let result = client.generate_embedding("").await; + assert!(result.is_err(), "empty input should return an error"); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_throw_for_empty_batch() { + let (client, model) = setup_embedding_client().await; + + let result = client.generate_embeddings(&[]).await; + assert!(result.is_err(), "empty batch should return an error"); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_generate_batch_embeddings() { + let (client, model) = setup_embedding_client().await; + + let response = client + .generate_embeddings(&[ + "The quick brown fox jumps over the lazy dog", + "Machine learning is a subset of artificial intelligence", + "The capital of France is Paris", + ]) + .await + .expect("batch embedding should succeed"); + + assert_eq!(response.data.len(), 3); + for (i, data) in response.data.iter().enumerate() { + assert_eq!(data.index, i as u32); + assert_eq!(data.embedding.len(), 1024); + } + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_generate_normalized_batch_embeddings() { + let (client, model) = setup_embedding_client().await; + + let response = client + .generate_embeddings(&["Hello world", "Goodbye world"]) + .await + .expect("batch embedding should succeed"); + + assert_eq!(response.data.len(), 2); + for data in &response.data { + let norm: f32 = data.embedding.iter().map(|v| v * v).sum::().sqrt(); + assert!( + (0.99_f32..=1.01_f32).contains(&norm), + "L2 norm {norm} not approximately 1.0" + ); + } + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_match_single_and_batch_results() { + let (client, model) = setup_embedding_client().await; + + let single = client + .generate_embedding("The capital of France is Paris") + .await + .expect("single embedding should succeed"); + + let batch = client + .generate_embeddings(&["The capital of France is Paris"]) + .await + .expect("batch embedding should succeed"); + + assert_eq!(batch.data.len(), 1); + for (a, b) in single.data[0] + .embedding + .iter() + .zip(batch.data[0].embedding.iter()) + { + assert_eq!(a, b); + } + + model.unload().await.expect("unload should succeed"); +} diff --git a/sdk_v2/rust/tests/integration/live_audio_test.rs b/sdk_v2/rust/tests/integration/live_audio_test.rs new file mode 100644 index 000000000..ee873840f --- /dev/null +++ b/sdk_v2/rust/tests/integration/live_audio_test.rs @@ -0,0 +1,113 @@ +use super::common; +use std::sync::Arc; +use tokio_stream::StreamExt; + +/// Generate synthetic PCM audio (440Hz sine wave, 16kHz, 16-bit mono). +fn generate_sine_wave_pcm(sample_rate: i32, duration_seconds: i32, frequency: f64) -> Vec { + let total_samples = (sample_rate * duration_seconds) as usize; + let mut pcm_bytes = vec![0u8; total_samples * 2]; // 16-bit = 2 bytes per sample + + for i in 0..total_samples { + let t = i as f64 / sample_rate as f64; + let sample = + (i16::MAX as f64 * 0.5 * (2.0 * std::f64::consts::PI * frequency * t).sin()) as i16; + pcm_bytes[i * 2] = (sample & 0xFF) as u8; + pcm_bytes[i * 2 + 1] = ((sample >> 8) & 0xFF) as u8; + } + + pcm_bytes +} + +// --- E2E streaming test with synthetic PCM audio --- + +#[tokio::test] +async fn live_streaming_e2e_with_synthetic_pcm_returns_valid_response() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + + // Try to get a nemotron or whisper model for audio streaming + let model = match catalog.get_model("nemotron").await { + Ok(m) => m, + Err(_) => match catalog.get_model(common::WHISPER_MODEL_ALIAS).await { + Ok(m) => m, + Err(_) => { + eprintln!("Skipping E2E test: no audio model available"); + return; + } + }, + }; + + if !model.is_cached().await.unwrap_or(false) { + eprintln!("Skipping E2E test: model not cached"); + return; + } + + model.load().await.expect("model.load() failed"); + + let audio_client = model.create_audio_client(); + let session = audio_client.create_live_transcription_session(); + + // Verify default settings + assert_eq!(session.settings.sample_rate, 16000); + assert_eq!(session.settings.channels, 1); + + if let Err(e) = session.start(None).await { + eprintln!("Skipping E2E test: could not start session: {e}"); + model.unload().await.ok(); + return; + } + + // Start collecting results in background (must start before pushing audio) + let mut stream = session.get_stream().await.expect("get_stream failed"); + + let results = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let stream_error: Arc>> = + Arc::new(tokio::sync::Mutex::new(None)); + let results_clone = Arc::clone(&results); + let error_clone = Arc::clone(&stream_error); + let read_task = tokio::spawn(async move { + while let Some(result) = stream.next().await { + match result { + Ok(r) => results_clone.lock().await.push(r), + Err(e) => { + *error_clone.lock().await = Some(format!("{e}")); + break; + } + } + } + }); + + // Generate ~2 seconds of synthetic PCM audio (440Hz sine wave) + let pcm_bytes = generate_sine_wave_pcm(16000, 2, 440.0); + + // Push audio in chunks (100ms each, matching typical mic callback size) + let chunk_size = 16000 / 10 * 2; // 100ms of 16-bit audio = 3200 bytes + for offset in (0..pcm_bytes.len()).step_by(chunk_size) { + let end = std::cmp::min(offset + chunk_size, pcm_bytes.len()); + session + .append(&pcm_bytes[offset..end], None) + .await + .expect("append failed"); + } + + // Stop session to flush remaining audio and complete the stream + session.stop(None).await.expect("stop failed"); + read_task.await.expect("read task failed"); + + // Verify no stream errors occurred + assert!( + stream_error.lock().await.is_none(), + "Stream produced an error: {:?}", + stream_error.lock().await + ); + + // Verify response attributes — synthetic audio may or may not produce text, + // but the response objects should be properly structured (C#-compatible envelope) + let results = results.lock().await; + for result in results.iter() { + assert!(!result.content.is_empty(), "content must not be empty"); + assert_eq!(result.content[0].text, result.content[0].transcript); + } + + model.unload().await.expect("model.unload() failed"); +} diff --git a/sdk_v2/rust/tests/integration/main.rs b/sdk_v2/rust/tests/integration/main.rs new file mode 100644 index 000000000..799446a95 --- /dev/null +++ b/sdk_v2/rust/tests/integration/main.rs @@ -0,0 +1,20 @@ +//! Single integration test binary for the Foundry Local Rust SDK. +//! +//! All test modules are compiled into one binary so the native core is only +//! initialised once (via the `OnceLock` singleton in `FoundryLocalManager`). +//! Running them as separate binaries causes "already initialized" errors +//! because the .NET native runtime retains state across process-level +//! library loads. +#![allow(deprecated)] // some suites still exercise the deprecated OpenAI facade + +mod common; + +mod audio_client_test; +mod catalog_test; +mod chat_client_test; +mod embedding_client_test; +mod live_audio_test; +mod manager_test; +mod model_test; +mod session_test; +mod web_service_test; diff --git a/sdk_v2/rust/tests/integration/manager_test.rs b/sdk_v2/rust/tests/integration/manager_test.rs new file mode 100644 index 000000000..aa3e06148 --- /dev/null +++ b/sdk_v2/rust/tests/integration/manager_test.rs @@ -0,0 +1,21 @@ +use super::common; +use foundry_local_sdk::FoundryLocalManager; + +#[test] +fn should_initialize_successfully() { + let config = common::test_config(); + let manager = FoundryLocalManager::create(config); + assert!( + manager.is_ok(), + "Manager creation failed: {:?}", + manager.err() + ); +} + +#[test] +fn should_return_catalog_with_non_empty_name() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let name = catalog.name(); + assert!(!name.is_empty(), "Catalog name should not be empty"); +} diff --git a/sdk_v2/rust/tests/integration/model_test.rs b/sdk_v2/rust/tests/integration/model_test.rs new file mode 100644 index 000000000..c1ffa171e --- /dev/null +++ b/sdk_v2/rust/tests/integration/model_test.rs @@ -0,0 +1,323 @@ +use super::common; +use std::sync::Arc; + +// ── Cached model verification ──────────────────────────────────────────────── + +#[tokio::test] +async fn should_verify_cached_models_from_test_data_shared() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let cached = catalog + .get_cached_models() + .await + .expect("get_cached_models failed"); + + let has_qwen = cached.iter().any(|m| m.alias() == common::TEST_MODEL_ALIAS); + assert!( + has_qwen, + "'{}' should be present in cached models", + common::TEST_MODEL_ALIAS + ); + + let has_whisper = cached + .iter() + .any(|m| m.alias() == common::WHISPER_MODEL_ALIAS); + assert!( + has_whisper, + "'{}' should be present in cached models", + common::WHISPER_MODEL_ALIAS + ); +} + +// ── Load / Unload ──────────────────────────────────────────────────────────── + +#[tokio::test] +async fn should_load_and_unload_model() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let model = catalog + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + model.load().await.expect("model.load() failed"); + assert!( + model.is_loaded().await.expect("is_loaded check failed"), + "Model should be loaded after load()" + ); + + model.unload().await.expect("model.unload() failed"); + assert!( + !model.is_loaded().await.expect("is_loaded check failed"), + "Model should not be loaded after unload()" + ); +} + +// ── Introspection ──────────────────────────────────────────────────────────── + +#[tokio::test] +async fn should_expose_alias() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + assert_eq!(model.alias(), common::TEST_MODEL_ALIAS); +} + +#[tokio::test] +async fn should_expose_non_empty_id() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + println!("Model id: {}", model.id()); + + assert!( + !model.id().is_empty(), + "Model id() should be a non-empty string" + ); +} + +#[tokio::test] +async fn should_have_at_least_one_variant() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + let variants = model.variants(); + println!("Model has {} variant(s)", variants.len()); + + assert!( + !variants.is_empty(), + "Model should have at least one variant" + ); +} + +#[tokio::test] +async fn should_have_selected_variant_matching_id() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + // The model's id() should return the selected variant's id + // info() delegates to the selected variant, so id() and info().id must agree + assert_eq!( + model.id(), + model.info().id, + "model.id() should match model.info().id (the selected variant's metadata)" + ); +} + +#[tokio::test] +async fn should_report_cached_model_as_cached() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + let cached = model.is_cached().await.expect("is_cached() should succeed"); + assert!( + cached, + "Test model '{}' should be cached (from test-data-shared)", + common::TEST_MODEL_ALIAS + ); +} + +#[tokio::test] +async fn should_return_non_empty_path_for_cached_model() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + let path = model.path().await.expect("path() should succeed"); + println!("Model path: {}", path.display()); + + assert!( + !path.as_os_str().is_empty(), + "Cached model should have a non-empty path" + ); +} + +#[tokio::test] +async fn should_select_variant_by_model() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + // Remember the original selection so we can restore it afterward. + let original_id = model.id().to_string(); + + let first_variant = model.variants()[0].clone(); + let first_variant_id = first_variant.id().to_string(); + model + .select_variant(&first_variant) + .expect("select_variant should succeed"); + assert_eq!( + model.id(), + first_variant_id, + "After select_variant, id() should match the selected variant" + ); + + // Restore the original variant so other tests sharing this + // model via the catalog are not affected. + model + .select_variant_by_id(&original_id) + .expect("restoring original variant should succeed"); +} + +#[tokio::test] +async fn should_select_variant_by_id() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + let original_id = model.id().to_string(); + + let first_variant_id = model.variants()[0].id().to_string(); + model + .select_variant_by_id(&first_variant_id) + .expect("select_variant_by_id should succeed"); + assert_eq!( + model.id(), + first_variant_id, + "After select_variant_by_id, id() should match the selected variant" + ); + + model + .select_variant_by_id(&original_id) + .expect("restoring original variant should succeed"); +} + +#[tokio::test] +async fn should_fail_to_select_unknown_variant() { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + + let result = model.select_variant_by_id("nonexistent-variant-id"); + assert!( + result.is_err(), + "select_variant_by_id with unknown ID should fail" + ); + + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("not found"), + "Error should mention 'not found': {err_msg}" + ); +} + +// ── Load manager (core interop) ────────────────────────────────────────────── + +async fn get_test_model() -> Arc { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + catalog + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed") +} + +#[tokio::test] +async fn should_load_model_using_core_interop() { + let model = get_test_model().await; + model.load().await.expect("model.load() failed"); + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_unload_model_using_core_interop() { + let model = get_test_model().await; + model.load().await.expect("model.load() failed"); + model.unload().await.expect("model.unload() failed"); +} + +#[tokio::test] +async fn should_list_loaded_models_using_core_interop() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + + let loaded = catalog + .get_loaded_models() + .await + .expect("catalog.get_loaded_models() failed"); + + let _ = loaded; +} + +#[tokio::test] +#[ignore = "requires running web service"] +async fn should_load_and_unload_model_using_external_service() { + if common::is_running_in_ci() { + eprintln!("Skipping external-service test in CI"); + return; + } + + let manager = common::get_test_manager(); + let model = get_test_model().await; + + manager + .start_web_service() + .await + .expect("start_web_service failed"); + + model + .load() + .await + .expect("load via external service failed"); + + model + .unload() + .await + .expect("unload via external service failed"); +} + +#[tokio::test] +#[ignore = "requires running web service"] +async fn should_list_loaded_models_using_external_service() { + if common::is_running_in_ci() { + eprintln!("Skipping external-service test in CI"); + return; + } + + let manager = common::get_test_manager(); + + manager + .start_web_service() + .await + .expect("start_web_service failed"); + + let catalog = manager.catalog(); + let loaded = catalog + .get_loaded_models() + .await + .expect("get_loaded_models via external service failed"); + + let _ = loaded; +} diff --git a/sdk_v2/rust/tests/integration/session_test.rs b/sdk_v2/rust/tests/integration/session_test.rs new file mode 100644 index 000000000..124773ea9 --- /dev/null +++ b/sdk_v2/rust/tests/integration/session_test.rs @@ -0,0 +1,170 @@ +//! Integration tests for the low-level inference API: `Session` / `ChatSession` +//! / `EmbeddingsSession` with `Request` / `Response` / `Item`. + +use std::sync::Arc; + +use foundry_local_sdk::{ + ChatSession, EmbeddingsSession, Item, MessageRole, Model, Request, RequestOptions, + SearchOptions, +}; +use tokio_stream::StreamExt; + +use super::common; + +async fn setup_chat_session() -> (ChatSession, Arc) { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + model.load().await.expect("model.load() failed"); + let session = ChatSession::new(&model) + .await + .expect("ChatSession::new failed"); + (session, model) +} + +async fn setup_embeddings_session() -> (EmbeddingsSession, Arc) { + let manager = common::get_test_manager(); + let model = manager + .catalog() + .get_model(common::EMBEDDING_MODEL_ALIAS) + .await + .expect("embedding model should exist in catalog"); + model.load().await.expect("model.load() failed"); + let session = EmbeddingsSession::new(&model) + .await + .expect("EmbeddingsSession::new failed"); + (session, model) +} + +/// Deterministic options: greedy decoding with a bounded output length. +fn deterministic_options() -> RequestOptions { + RequestOptions { + search: SearchOptions { + temperature: Some(0.0), + max_output_tokens: Some(500), + ..Default::default() + }, + ..Default::default() + } +} + +#[tokio::test] +async fn should_process_request_and_return_response() { + let (session, model) = setup_chat_session().await; + + let request = Request::from_items(vec![ + Item::message( + MessageRole::System, + vec![Item::text( + "You are a helpful math assistant. Respond with just the answer.", + )], + ), + Item::user_message(vec![Item::text("What is 7*6?")]), + ]) + .with_options(deterministic_options()); + + let response = session + .process_request(request) + .await + .expect("process_request failed"); + + let text = response.text(); + println!("Response: {text}"); + assert!( + text.contains("42"), + "Expected response to contain '42', got: {text}" + ); + assert!( + !response.items.is_empty(), + "response should contain at least one item" + ); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_stream_items() { + let (session, model) = setup_chat_session().await; + + let request = Request::from_items(vec![Item::user_message(vec![Item::text("What is 7*6?")])]) + .with_options(deterministic_options()); + + let mut stream = session.process_streaming_request(request); + let mut collected = String::new(); + while let Some(item) = stream.next().await { + let item = item.expect("stream item failed"); + if let Some(text) = item.as_text() { + collected.push_str(text); + } + } + + println!("Streamed: {collected}"); + assert!( + collected.contains("42"), + "Expected streamed text to contain '42', got: {collected}" + ); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_track_turn_count() { + let (session, model) = setup_chat_session().await; + assert_eq!(session.turn_count(), 0, "new session should have 0 turns"); + + let request = Request::from_items(vec![Item::user_message(vec![Item::text("Say hi.")])]) + .with_options(deterministic_options()); + session + .process_request(request) + .await + .expect("process_request failed"); + + assert!( + session.turn_count() >= 1, + "turn count should advance after a processed request, got {}", + session.turn_count() + ); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_embed_text() { + let (session, model) = setup_embeddings_session().await; + + let embedding = session + .embed("The quick brown fox jumps over the lazy dog") + .await + .expect("embed failed"); + + assert_eq!(embedding.len(), 1024, "unexpected embedding dimension"); + let norm: f32 = embedding.iter().map(|v| v * v).sum::().sqrt(); + assert!(norm > 0.0, "embedding should be a non-zero vector"); + + model.unload().await.expect("unload should succeed"); +} + +#[tokio::test] +async fn should_embed_batch() { + let (session, model) = setup_embeddings_session().await; + + let inputs = vec![ + "The quick brown fox jumps over the lazy dog".to_string(), + "Machine learning is a subset of artificial intelligence".to_string(), + "The capital of France is Paris".to_string(), + ]; + let vectors = session + .embed_batch(inputs.clone()) + .await + .expect("embed_batch failed"); + + assert_eq!(vectors.len(), inputs.len(), "one vector per input"); + for vector in &vectors { + assert_eq!(vector.len(), 1024, "unexpected embedding dimension"); + } + + model.unload().await.expect("unload should succeed"); +} diff --git a/sdk_v2/rust/tests/integration/web_service_test.rs b/sdk_v2/rust/tests/integration/web_service_test.rs new file mode 100644 index 000000000..9222f9d45 --- /dev/null +++ b/sdk_v2/rust/tests/integration/web_service_test.rs @@ -0,0 +1,161 @@ +use super::common; +use serde_json::json; + +/// Start the web service, make a non-streaming POST to v1/chat/completions, +/// verify we get a valid response, then stop the service. +#[tokio::test] +async fn should_complete_chat_via_rest_api() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let model = catalog + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + model.load().await.expect("model.load() failed"); + + manager + .start_web_service() + .await + .expect("start_web_service failed"); + let urls = manager.urls().expect("urls() should succeed"); + let base_url = urls.first().expect("no URL returned").trim_end_matches('/'); + + let client = reqwest::Client::new(); + let resp = client + .post(format!("{base_url}/v1/chat/completions")) + .json(&json!({ + "model": model.id(), + "messages": [ + { "role": "system", "content": "You are a helpful math assistant. Respond with just the answer." }, + { "role": "user", "content": "What is 7*6?" } + ], + "max_tokens": 500, + "temperature": 0.0, + "stream": false + })) + .send() + .await + .expect("HTTP request failed"); + + assert!( + resp.status().is_success(), + "Expected 2xx, got {}", + resp.status() + ); + + let body: serde_json::Value = resp.json().await.expect("failed to parse response JSON"); + let content = body + .pointer("/choices/0/message/content") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + println!("REST response: {content}"); + + assert!( + content.contains("42"), + "Expected response to contain '42', got: {content}" + ); + + manager + .stop_web_service() + .await + .expect("stop_web_service failed"); + model.unload().await.expect("model.unload() failed"); +} + +/// Start the web service, make a streaming POST to v1/chat/completions, +/// collect SSE chunks, verify we get a valid streamed response. +#[tokio::test] +async fn should_stream_chat_via_rest_api() { + let manager = common::get_test_manager(); + let catalog = manager.catalog(); + let model = catalog + .get_model(common::TEST_MODEL_ALIAS) + .await + .expect("get_model failed"); + model.load().await.expect("model.load() failed"); + + manager + .start_web_service() + .await + .expect("start_web_service failed"); + let urls = manager.urls().expect("urls() should succeed"); + let base_url = urls.first().expect("no URL returned").trim_end_matches('/'); + + let client = reqwest::Client::new(); + let mut response = client + .post(format!("{base_url}/v1/chat/completions")) + .json(&json!({ + "model": model.id(), + "messages": [ + { "role": "system", "content": "You are a helpful math assistant. Respond with just the answer." }, + { "role": "user", "content": "What is 7*6?" } + ], + "max_tokens": 500, + "temperature": 0.0, + "stream": true + })) + .send() + .await + .expect("HTTP request failed"); + + assert!( + response.status().is_success(), + "Expected 2xx, got {}", + response.status() + ); + + let mut full_text = String::new(); + while let Some(chunk) = response.chunk().await.expect("chunk read failed") { + let text = String::from_utf8_lossy(&chunk); + for line in text.lines() { + let line = line.trim(); + if let Some(data) = line.strip_prefix("data: ") { + if data == "[DONE]" { + break; + } + if let Ok(parsed) = serde_json::from_str::(data) { + if let Some(content) = parsed + .pointer("/choices/0/delta/content") + .and_then(|v| v.as_str()) + { + full_text.push_str(content); + } + } + } + } + } + + println!("REST streamed response: {full_text}"); + + assert!( + full_text.contains("42"), + "Expected streamed response to contain '42', got: {full_text}" + ); + + manager + .stop_web_service() + .await + .expect("stop_web_service failed"); + model.unload().await.expect("model.unload() failed"); +} + +/// urls() should return the listening addresses after start_web_service. +#[tokio::test] +async fn should_expose_urls_after_start() { + let manager = common::get_test_manager(); + + manager + .start_web_service() + .await + .expect("start_web_service failed"); + + let urls = manager.urls().expect("urls() should succeed"); + println!("Web service URLs: {urls:?}"); + assert!(!urls.is_empty(), "urls() should return URLs after start"); + + manager + .stop_web_service() + .await + .expect("stop_web_service failed"); +}