diff --git a/.cursorignore b/.cursorignore deleted file mode 100644 index 96e1c6e1..00000000 --- a/.cursorignore +++ /dev/null @@ -1,8 +0,0 @@ -# Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv) - -assets/ -benchmarks/ -examples/ -openarc_bench.db -openarc.log -scratchpad.md \ No newline at end of file diff --git a/Dockerfile b/.devops/openvino.Dockerfile similarity index 97% rename from Dockerfile rename to .devops/openvino.Dockerfile index d49b5ee7..251d8880 100644 --- a/Dockerfile +++ b/.devops/openvino.Dockerfile @@ -6,8 +6,8 @@ # - battlemage : Battlemage Intel GPU path # # Build examples: -# docker build --target standard -t {imagename}:dev . -# docker build --target battlemage -t {imagename}-battlemage:dev . +# docker build --target standard -t {imagename}:dev -f .devops/openvino.Dockerfile . +# docker build --target battlemage -t {imagename}-battlemage:dev -f .devops/openvino.Dockerfile . # ============================================================================ # ============================================================================ diff --git a/.github/workflows/build-container-image.yaml b/.github/workflows/build-container-image.yaml index 65627ffc..d7daeb72 100644 --- a/.github/workflows/build-container-image.yaml +++ b/.github/workflows/build-container-image.yaml @@ -56,6 +56,7 @@ jobs: with: context: . target: standard + file: .devops/openvino.Dockerfile push: true tags: ${{ steps.meta-standard.outputs.tags }} labels: ${{ steps.meta-standard.outputs.labels }} @@ -80,6 +81,7 @@ jobs: with: context: . target: battlemage + file: .devops/openvino.Dockerfile push: true tags: ${{ steps.meta-battlemage.outputs.tags }} labels: ${{ steps.meta-battlemage.outputs.labels }} diff --git a/AGENTS.md b/AGENTS.md index 6360a051..6f462ab8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -OpenArc uses bleeding edge libraries and APIs you may not be familair with. When working through a task, use the deepwiki mcp server to get *contextual* information, and the command line to investigate python surfaces. Prefer the command line. +OpenArc uses bleeding edge libraries and APIs you may not be familair with. Ask the user for access to reference matieral. diff --git a/benchmark/context_overflow.py b/benchmark/context_overflow.py deleted file mode 100644 index 87460ff4..00000000 --- a/benchmark/context_overflow.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -""" -Context Overflow Test Script - -Tests increasing context sizes with OpenAI API-compatible endpoint. -Progressively increases prompt size each iteration and accumulates context. -Relies on server-reported metrics for accurate token counts. -""" - -import argparse -import os -import sys -from openai import OpenAI - - -def generate_filler_text(char_count: int) -> str: - """Generate filler text for approximately the given character count.""" - filler = "The quick brown fox jumps over the lazy dog. " * ((char_count // 45) + 1) - return filler[:char_count] - - -def main(): - parser = argparse.ArgumentParser( - description="Test context overflow with increasing prompts" - ) - parser.add_argument( - "--model", - type=str, - required=True, - help="Model name to use for testing" - ) - parser.add_argument( - "--iterations", - type=int, - default=10, - help="Number of iterations to run (default: 10)" - ) - args = parser.parse_args() - - # Get API key from environment - api_key = os.environ.get("OPENARC_API_KEY") - if not api_key: - print("ERROR: OPENARC_API_KEY environment variable not set", file=sys.stderr) - sys.exit(1) - - # Initialize OpenAI client pointing to localhost:8000 - client = OpenAI( - api_key=api_key, - base_url="http://localhost:8000/v1" - ) - - # Configuration - char_increment = 4096 # Add roughly ~2048 tokens worth of characters per iteration - accumulated_context = "" - base_prompt = "...then mistral says, youre an impasta!:\n\n" - - print(f"Model: {args.model}") - print(f"Iterations: {args.iterations}") - print(f"Character increment per iteration: {char_increment}") - print(f"{'='*40}\n") - - for iteration in range(1, args.iterations + 1): - # Add more context each iteration - filler = generate_filler_text(char_increment) - accumulated_context += filler - - # Build the full prompt - full_prompt = base_prompt + accumulated_context + "\nTell 100 more new jokes" - - print(f"Iteration {iteration}/{args.iterations}") - - try: - response = client.chat.completions.create( - model=args.model, - messages=[ - { - "role": "user", - "content": full_prompt - } - ], - max_tokens=128 - ) - - # Print response details - if response.choices: - content = response.choices[0].message.content - print(" ✓ Success") - print(f" Response preview: {content[:80]}...") - - # Extract and display OpenArc internal metrics - metrics = getattr(response, 'metrics', None) - if metrics: - print("\n === Performance Metrics ===") - if 'ttft (s)' in metrics: - print(f" TTFT: {metrics['ttft (s)']:.2f}s") - if 'tpot (ms)' in metrics: - print(f" TPOT: {metrics['tpot (ms)']:.2f}ms") - if 'prefill_throughput (tokens/s)' in metrics: - print(f" Prefill throughput: {metrics['prefill_throughput (tokens/s)']:.2f} tokens/s") - if 'decode_throughput (tokens/s)' in metrics: - print(f" Decode throughput: {metrics['decode_throughput (tokens/s)']:.2f} tokens/s") - if 'decode_duration (s)' in metrics: - print(f" Decode duration: {metrics['decode_duration (s)']:.2f}s") - - # Use server-reported token counts - if response.usage: - prompt_tokens = response.usage.prompt_tokens - completion_tokens = response.usage.completion_tokens - total_tokens = response.usage.total_tokens - - print(f"\n Prompt tokens: {prompt_tokens:,}") - print(f" Completion tokens: {completion_tokens:,}") - print(f" Total tokens: {total_tokens:,}") - - except Exception as e: - print(f" ✗ Error: {type(e).__name__}: {str(e)}") - break - - print() - - print("=== Test Complete ===") - - -if __name__ == "__main__": - main() diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..f953a31e --- /dev/null +++ b/config.yaml @@ -0,0 +1,57 @@ +server: +port: + + +# comma seperated list of model_name to load on startup + +# from env +startup_models : ${OPENRARC_AUTOLOAD_MODELS} + + +models: + qwen35-08b: + engine: ovgenai + model_type: llm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.5-0.8B-int8_asym-ov/ + device: CPU + runtime_config: + PERFORMANCE_HINT: LATENCY + samplers: + temperature: 0.7 + top_k: 40 + top_p: 0.95 + repetition_penalty: 1.05 + max_tokens: 1024 + + + kokoro: + engine: openvino + model_type: kokoro + model_path: + device: CPU + kokoro_options: + voice: af_sarah + voice_blend: af_heart:0.7,af_nicole:0.3 + lang_code: + speed: 1.0 + chracter_count_chunk: 100 + + qwen3asr: + engine: openvino + model_type: qwen3_asr + model_path: + device: CPU + qwen3_asr_options: + language: + max_tokens: + max_chunk_sec: + search_expand_sec: + min_window_ms: + + + qwen3_tts_elmo: + model_type: qwen3_tts_voice_clone + model_path: + engine: openvino + device: GPU.0 + diff --git a/docs/configure.md b/docs/configure.md new file mode 100644 index 00000000..bf8d7438 --- /dev/null +++ b/docs/configure.md @@ -0,0 +1,24 @@ +runtime_config + + +runtime_config is an OpenArc entrypoint to the *properties* way of configuring openvino runtime. These settings allow users to tune the behavior of openivno runtime without needing to change application logic and are meant to be "portable", requring no code changes. Since OpenArc + +OpenArc does not validate these, and OpenVINO upstream does not provide a way to check the behvaior of these settings in all cases. They can help you access hardware features not available to all devices like `SCHEDULING_CORE_TYPE` for more recent Intel CPUs, debug numeircal precision issues with `INFERENCE_PRECISION_HINT` or control `KV_CACHE_PRECISION`. + +*properties* have the worst documentation in all of OpenVINO ecosystem, yet they are used everywhere in the openvino_notebooks, PRs and sometimes are even hardcoded depending on the needs of OpenVINO team. In that way, poking at these settings can drastically change performance but have less knobs than users of projects like `llama.cpp`, `vllm`, `sglang` are familiar with making. + + +Even though we can learn from the source code what these settings do knowing when they are useful comes with practice + + +ATTENTION_BACKEND "SDPA", "PA" +KV_CACHE_PRECISION "u4", "u8", "f16", "f32" +PERFORMANCE_HINT "LATNENCY", "THROUGHPUT" +EXECUTION_MODE_HINT "ACCURACY", "PERFORMANCE" +INFERENCE_PRECISION_HINT "f16", "f32" +MODEL_DISTRIBUTION_POLICY "TENSOR_PARALLEL", "PIPELINE_PARALLEL" +ACTIVATIONS_SCALING_FACTOR: +DYNAMIC_QUANTIZATION_GROUP_SIZE: integer +ENABLE_HYPER_THREADING: bool, defaults to true +SCHEDULING_CORE_TYPE: "ANY_CORE", "ECORE_ONLY", "PCORE_ONLY" +LOG_LEVEL: "ERR", "WARN", "INFO", "DEBUG", "TRACE" # might require building openvino \ No newline at end of file diff --git a/gpu-metrics/build/temp.linux-x86_64-cpython-312/gpu_metrics.o b/gpu-metrics/build/temp.linux-x86_64-cpython-312/gpu_metrics.o deleted file mode 100644 index 445890d2..00000000 Binary files a/gpu-metrics/build/temp.linux-x86_64-cpython-312/gpu_metrics.o and /dev/null differ diff --git a/src/engine/openvino/kokoro.py b/src/engine/openvino/kokoro.py index 3d80ac83..c905c98a 100644 --- a/src/engine/openvino/kokoro.py +++ b/src/engine/openvino/kokoro.py @@ -19,8 +19,8 @@ from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig -from src.server.models.openvino import OV_KokoroGenConfig +from src.server.schemas.registration import ModelLoadConfig +from src.server.schemas.modeling.contract_kokoro import OV_KokoroGenConfig class StreamChunk(NamedTuple): diff --git a/src/engine/openvino/qwen3_asr/qwen3_asr.py b/src/engine/openvino/qwen3_asr/qwen3_asr.py index 412daa83..c95e4e0e 100644 --- a/src/engine/openvino/qwen3_asr/qwen3_asr.py +++ b/src/engine/openvino/qwen3_asr/qwen3_asr.py @@ -36,9 +36,9 @@ validate_language, ) -from src.server.models.openvino import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig from src.server.model_registry import ModelRegistry -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType logger = logging.getLogger(__name__) diff --git a/src/engine/openvino/qwen3_tts/qwen3_tts.py b/src/engine/openvino/qwen3_tts/qwen3_tts.py index f24b98ec..7b0a751d 100644 --- a/src/engine/openvino/qwen3_tts/qwen3_tts.py +++ b/src/engine/openvino/qwen3_tts/qwen3_tts.py @@ -46,8 +46,13 @@ _SYNTH_TMPL, ) from src.server.model_registry import ModelRegistry -from src.server.models.openvino import OV_Qwen3TTSGenConfig -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_qwen3tts import ( + OV_Qwen3TTSCustomVoice, + OV_Qwen3TTSVoiceClone, + OV_Qwen3TTSVoiceDesign, + OV_Qwen3TTSGenConfig, +) +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType logger = logging.getLogger(__name__) @@ -198,27 +203,33 @@ def generate_stream(self, gen_config: OV_Qwen3TTSGenConfig) -> Iterator[TTSStrea if not self._loaded: raise RuntimeError("Call load_model() before generate_stream()") gc = gen_config.model_copy(update={"non_streaming_mode": False}) - if self.load_config.model_type == ModelType.QWEN3_TTS_VOICE_CLONE: + if isinstance(gc, OV_Qwen3TTSVoiceClone): yield from self._generate_voice_clone_stream(gc) - else: + elif isinstance(gc, (OV_Qwen3TTSCustomVoice, OV_Qwen3TTSVoiceDesign)): yield from self._generate_standard_stream(gc) + else: + raise TypeError(f"Unsupported gen_config type: {type(gc).__name__}") def _generate_sync(self, gen_config: OV_Qwen3TTSGenConfig) -> tuple[np.ndarray, int]: if not self._loaded: raise RuntimeError("Call load_model() before generate()") - if self.load_config.model_type == ModelType.QWEN3_TTS_VOICE_CLONE: + if isinstance(gen_config, OV_Qwen3TTSVoiceClone): return self._generate_voice_clone(gen_config) - return self._generate_standard(gen_config) + if isinstance(gen_config, (OV_Qwen3TTSCustomVoice, OV_Qwen3TTSVoiceDesign)): + return self._generate_standard(gen_config) + raise TypeError(f"Unsupported gen_config type: {type(gen_config).__name__}") # ---- Internal: standard generation (custom_voice / voice_design) -------- - def _generate_standard(self, gen_config: OV_Qwen3TTSGenConfig) -> tuple[np.ndarray, int]: + def _generate_standard( + self, gen_config: OV_Qwen3TTSCustomVoice | OV_Qwen3TTSVoiceDesign, + ) -> tuple[np.ndarray, int]: t_total = time.perf_counter() perf: dict = {} - speaker = Speaker(gen_config.speaker) if gen_config.speaker else None language = Language(gen_config.language) if gen_config.language else None - if self.load_config.model_type == ModelType.QWEN3_TTS_CUSTOM_VOICE: + if isinstance(gen_config, OV_Qwen3TTSCustomVoice): + speaker = Speaker(gen_config.speaker) if gen_config.speaker else None build_kw = dict( text=gen_config.input, speaker=speaker, @@ -251,7 +262,7 @@ def _generate_standard(self, gen_config: OV_Qwen3TTSGenConfig) -> tuple[np.ndarr # ---- Internal: voice clone generation ----------------------------------- - def _generate_voice_clone(self, gen_config: OV_Qwen3TTSGenConfig) -> tuple[np.ndarray, int]: + def _generate_voice_clone(self, gen_config: OV_Qwen3TTSVoiceClone) -> tuple[np.ndarray, int]: t_total = time.perf_counter() perf: dict = {} language = Language(gen_config.language) if gen_config.language else None @@ -303,13 +314,15 @@ def _generate_voice_clone(self, gen_config: OV_Qwen3TTSGenConfig) -> tuple[np.nd self._log_pipeline_summary(perf, t_total, wav_seconds=float(len(wav) / SPEECH_DECODER_SR), voice_clone=True) return wav, SPEECH_DECODER_SR - def _generate_standard_stream(self, gen_config: OV_Qwen3TTSGenConfig) -> Iterator[TTSStreamChunk]: + def _generate_standard_stream( + self, gen_config: OV_Qwen3TTSCustomVoice | OV_Qwen3TTSVoiceDesign, + ) -> Iterator[TTSStreamChunk]: t_total = time.perf_counter() perf: dict = {} - speaker = Speaker(gen_config.speaker) if gen_config.speaker else None language = Language(gen_config.language) if gen_config.language else None - if self.load_config.model_type == ModelType.QWEN3_TTS_CUSTOM_VOICE: + if isinstance(gen_config, OV_Qwen3TTSCustomVoice): + speaker = Speaker(gen_config.speaker) if gen_config.speaker else None build_kw = dict( text=gen_config.input, speaker=speaker, @@ -340,7 +353,7 @@ def _generate_standard_stream(self, gen_config: OV_Qwen3TTSGenConfig) -> Iterato if n_samples > 0: logger.info(f"[info] streaming: {n_chunks} chunks -> {n_samples} samples ({wav_sec:.2f}s audio)") - def _generate_voice_clone_stream(self, gen_config: OV_Qwen3TTSGenConfig) -> Iterator[TTSStreamChunk]: + def _generate_voice_clone_stream(self, gen_config: OV_Qwen3TTSVoiceClone) -> Iterator[TTSStreamChunk]: t_total = time.perf_counter() perf: dict = {} language = Language(gen_config.language) if gen_config.language else None @@ -1090,7 +1103,7 @@ def _log_summary(codes: list, wav: np.ndarray, t_total_start: float): device=_device, runtime_config={}, ) - _gen = OV_Qwen3TTSGenConfig( + _gen = OV_Qwen3TTSVoiceClone( input=_synth_text, ref_audio_b64=_ref_b64, ref_text=_ref_text, diff --git a/src/engine/optimum/optimum_emb.py b/src/engine/optimum/optimum_emb.py index bf8b429f..fa9f8fdd 100755 --- a/src/engine/optimum/optimum_emb.py +++ b/src/engine/optimum/optimum_emb.py @@ -12,12 +12,12 @@ from transformers import AutoTokenizer from optimum.intel import OVModelForFeatureExtraction -from src.server.models.optimum import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig from typing import Any, AsyncIterator, Dict from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig +from src.server.schemas.registration import ModelLoadConfig diff --git a/src/engine/optimum/optimum_rr.py b/src/engine/optimum/optimum_rr.py index 0af5e280..7da23b1e 100644 --- a/src/engine/optimum/optimum_rr.py +++ b/src/engine/optimum/optimum_rr.py @@ -9,9 +9,9 @@ from transformers import AutoTokenizer from optimum.intel import OVModelForCausalLM -from src.server.models.optimum import RerankerConfig +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig +from src.server.schemas.registration import ModelLoadConfig class Optimum_RR: diff --git a/src/engine/ov_genai/llm.py b/src/engine/ov_genai/llm.py index 126757c4..c0f008d3 100755 --- a/src/engine/ov_genai/llm.py +++ b/src/engine/ov_genai/llm.py @@ -12,9 +12,9 @@ ) from transformers import AutoTokenizer, BatchEncoding -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig +from src.server.schemas.registration import ModelLoadConfig from src.engine.ov_genai.streamers import ChunkStreamer from src.server.utils.chat import flatten_messages diff --git a/src/engine/ov_genai/streamers.py b/src/engine/ov_genai/streamers.py index 699d7bbe..226f09a9 100644 --- a/src/engine/ov_genai/streamers.py +++ b/src/engine/ov_genai/streamers.py @@ -3,7 +3,7 @@ import asyncio from openvino_genai import StreamerBase -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig class ChunkStreamer(StreamerBase): diff --git a/src/engine/ov_genai/vlm.py b/src/engine/ov_genai/vlm.py index aff294f6..a8b23bb8 100644 --- a/src/engine/ov_genai/vlm.py +++ b/src/engine/ov_genai/vlm.py @@ -1,6 +1,7 @@ import asyncio import base64 import gc +import os import logging from io import BytesIO @@ -15,11 +16,11 @@ from PIL import Image from transformers import AutoTokenizer -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig from src.server.utils.chat import flatten_message_content from src.server.utils.resolve_vlm_type import is_qwen3_5_architecture, resolve_vlm_vision_token from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig +from src.server.schemas.registration import ModelLoadConfig from src.engine.ov_genai.streamers import ChunkStreamer logger = logging.getLogger(__name__) diff --git a/src/engine/ov_genai/whisper.py b/src/engine/ov_genai/whisper.py index 9d722816..db5814c5 100644 --- a/src/engine/ov_genai/whisper.py +++ b/src/engine/ov_genai/whisper.py @@ -10,8 +10,8 @@ from openvino_genai import WhisperPipeline from src.server.model_registry import ModelRegistry -from src.server.models.registration import ModelLoadConfig -from src.server.models.ov_genai import OVGenAI_WhisperGenConfig +from src.server.schemas.registration import ModelLoadConfig +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig logger = logging.getLogger(__name__) diff --git a/src/server/main.py b/src/server/main.py index d965142e..7b73af9f 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -16,7 +16,7 @@ from starlette.middleware.base import BaseHTTPMiddleware from src.server.deps import _registry -from src.server.models.registration import ModelLoadConfig +from src.server.schemas.registration import ModelLoadConfig from src.server.routes.openai import router as openai_router from src.server.routes.openarc import router as openarc_router diff --git a/src/server/model_registry.py b/src/server/model_registry.py index 2454fd48..7dbbfcff 100644 --- a/src/server/model_registry.py +++ b/src/server/model_registry.py @@ -9,7 +9,7 @@ from datetime import datetime from typing import Any, Awaitable, Callable, Dict, List, Optional, Set -from src.server.models.registration import ( +from src.server.schemas.registration import ( EngineType, ModelLoadConfig, ModelStatus, diff --git a/src/server/models/openvino.py b/src/server/models/openvino.py deleted file mode 100644 index 49959f67..00000000 --- a/src/server/models/openvino.py +++ /dev/null @@ -1,200 +0,0 @@ - -from enum import Enum -from pydantic import BaseModel, Field, field_validator -from typing import Optional - - - -class KokoroLanguage(str, Enum): - """Language codes for Kokoro TTS voices""" - AMERICAN_ENGLISH = "a" - BRITISH_ENGLISH = "b" - JAPANESE = "j" - MANDARIN_CHINESE = "z" - SPANISH = "e" - FRENCH = "f" - HINDI = "h" - ITALIAN = "i" - BRAZILIAN_PORTUGUESE = "p" - -class KokoroVoice(str, Enum): - """Available Kokoro TTS voices organized by language""" - # American English (🇺🇸) - 11F 9M - AF_HEART = "af_heart" - AF_ALLOY = "af_alloy" - AF_AOEDE = "af_aoede" - AF_BELLA = "af_bella" - AF_JESSICA = "af_jessica" - AF_KORE = "af_kore" - AF_NICOLE = "af_nicole" - AF_NOVA = "af_nova" - AF_RIVER = "af_river" - AF_SARAH = "af_sarah" - AF_SKY = "af_sky" - AM_ADAM = "am_adam" - AM_ECHO = "am_echo" - AM_ERIC = "am_eric" - AM_FENRIR = "am_fenrir" - AM_LIAM = "am_liam" - AM_MICHAEL = "am_michael" - AM_ONYX = "am_onyx" - AM_PUCK = "am_puck" - AM_SANTA = "am_santa" - - # British English (🇬🇧) - 4F 4M - BF_ALICE = "bf_alice" - BF_EMMA = "bf_emma" - BF_ISABELLA = "bf_isabella" - BF_LILY = "bf_lily" - BM_DANIEL = "bm_daniel" - BM_FABLE = "bm_fable" - BM_GEORGE = "bm_george" - BM_LEWIS = "bm_lewis" - - # Japanese (🇯🇵) - 4F 1M - JF_ALPHA = "jf_alpha" - JF_GONGITSUNE = "jf_gongitsune" - JF_NEZUMI = "jf_nezumi" - JF_TEBUKURO = "jf_tebukuro" - JM_KUMO = "jm_kumo" - - # Mandarin Chinese (🇨🇳) - 4F 4M - ZF_XIAOBEI = "zf_xiaobei" - ZF_XIAONI = "zf_xiaoni" - ZF_XIAOXIAO = "zf_xiaoxiao" - ZF_XIAOYI = "zf_xiaoyi" - ZM_YUNJIAN = "zm_yunjian" - ZM_YUNXI = "zm_yunxi" - ZM_YUNXIA = "zm_yunxia" - ZM_YUNYANG = "zm_yunyang" - - # Spanish (🇪🇸) - 1F 2M - EF_DORA = "ef_dora" - EM_ALEX = "em_alex" - EM_SANTA = "em_santa" - - # French (🇫🇷) - 1F - FF_SIWIS = "ff_siwis" - - # Hindi (🇮🇳) - 2F 2M - HF_ALPHA = "hf_alpha" - HF_BETA = "hf_beta" - HM_OMEGA = "hm_omega" - HM_PSI = "hm_psi" - - # Italian (🇮🇹) - 1F 1M - IF_SARA = "if_sara" - IM_NICOLA = "im_nicola" - - # Brazilian Portuguese (🇧🇷) - 1F 2M - PF_DORA = "pf_dora" - PM_ALEX = "pm_alex" - PM_SANTA = "pm_santa" - -class OV_KokoroGenConfig(BaseModel): - input: Optional[str] = Field(default=None, description="Injected from top-level request.input by the handler; do not set here.") - voice: KokoroVoice = Field(KokoroVoice.AF_SARAH, description="Voice token from available Kokoro voices") - # Optional weighted blend of voicepacks. Overrides `voice` when set. - # Format: "af_heart,af_nicole" (equal weights) or - # "af_heart:0.7,af_nicole:0.3" (weights normalised by engine). - voice_blend: Optional[str] = Field( - default=None, - description="Optional weighted blend of voicepacks, e.g. 'af_heart:0.7,af_nicole:0.3'. Overrides `voice`.", - ) - lang_code: KokoroLanguage = Field(KokoroLanguage.AMERICAN_ENGLISH, description="Language code for the voice") - speed: float = Field(1.0, description="Speech speed multiplier") - character_count_chunk: int = Field(100, description="Max characters per chunk") - response_format: str = Field("wav", description="Output format") - - @field_validator("voice_blend") - @classmethod - def _validate_voice_blend(cls, v: Optional[str]) -> Optional[str]: - if v is None or not v.strip(): - return None - parts = [p.strip() for p in v.split(",") if p.strip()] - if not parts: - return None - valid = {item.value for item in KokoroVoice} - for part in parts: - name, _, weight = part.partition(":") - name = name.strip() - if name not in valid: - raise ValueError(f"Unknown voice in blend: {name!r}") - if weight.strip(): - try: - w = float(weight) - except ValueError as exc: - raise ValueError(f"Invalid weight in blend for {name!r}: {weight!r}") from exc - if w < 0: - raise ValueError(f"Negative weight not allowed for {name!r}: {w}") - return v - - - - -class OV_Qwen3ASRGenConfig(BaseModel): - audio_base64: str | None = Field(default=None, description="Base64 encoded audio payload (injected from file when omitted)") - language: Optional[str] = Field(default=None, description="Optional forced language") - max_tokens: int = Field(default=1024, description="Maximum generated tokens per chunk") - max_chunk_sec: float = Field(default=30.0, description="Chunk size upper bound in seconds") - search_expand_sec: float = Field(default=5.0, description="Boundary search expansion in seconds") - min_window_ms: float = Field(default=100.0, description="Energy window in milliseconds") - - @field_validator("max_tokens") - @classmethod - def _validate_max_tokens(cls, v: int) -> int: - if v <= 0: - raise ValueError("max_tokens must be positive") - return v - - @field_validator("max_chunk_sec", "search_expand_sec", "min_window_ms") - @classmethod - def _validate_positive_float(cls, v: float) -> float: - if v <= 0: - raise ValueError("numeric values must be positive") - return v - -class OV_Qwen3TTSGenConfig(BaseModel): - """Single source of truth for all OVQwen3TTS request parameters. - - The model_type on ModelLoadConfig determines which mode the engine runs; - supply only the fields relevant to that mode: - - - qwen3_tts_custom_voice : input, speaker, language, instruct - - qwen3_tts_voice_design : input, voice_description, language - - qwen3_tts_voice_clone : input, ref_audio_b64, ref_text, x_vector_only, language, instruct - - All modes accept the sampling fields. - """ - # --- content --- - input: Optional[str] = Field(default=None, description="Injected from top-level request.input by the handler; do not set here.") - # [custom_voice] - speaker: str | None = Field(default=None, description="[custom_voice] Predefined speaker name.") - instruct: str | None = Field(default=None, description="[custom_voice, voice_clone] Optional style instruction.") - # [all] - language: str | None = Field(default=None, description="[all] Force output language. None = auto-detect.") - # [voice_design] - voice_description: str | None = Field(default=None, description="[voice_design] Free-form voice description.") - # [voice_clone] - ref_audio_b64: str | None = Field(default=None, description="[voice_clone] Base64-encoded reference WAV.") - ref_text: str | None = Field(default=None, description="[voice_clone] Transcript of reference audio (enables ICL).") - x_vector_only: bool = Field(default=False, description="[voice_clone] Use x-vector embedding only; skip ICL even if ref_text is set.") - # --- sampling (all modes) --- - max_new_tokens: int = Field(default=2048, description="Maximum codec frames to generate.") - do_sample: bool = Field(default=True, description="Sample from logits. False = greedy.") - top_k: int = Field(default=50, description="Top-k filter for talker logits.") - top_p: float = Field(default=1.0, description="Nucleus filter for talker logits. 1.0 = off.") - temperature: float = Field(default=0.9, description="Temperature scaling for talker logits.") - repetition_penalty: float = Field(default=1.05, description="Repetition penalty on first-codebook history. 1.0 = off.") - non_streaming_mode: bool = Field(default=True, description="True = all text tokens in prefill; False = drip-fed during decode.") - subtalker_do_sample: bool = Field(default=True, description="Sample sub-codebook logits.") - subtalker_top_k: int = Field(default=50, description="Top-k for code predictor.") - subtalker_top_p: float = Field(default=1.0, description="Nucleus filter for code predictor.") - subtalker_temperature: float = Field(default=0.9, description="Temperature for code predictor.") - # --- streaming (HTTP: audio/L16 chunked response when stream=True) --- - - # defaults taken from https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py#L886 - # these apply only for the 12.5hz tokenizer model. - stream: bool = Field(default=True, description="Enable streaming audio output (chunked PCM).") - stream_chunk_frames: int = Field(default=300, description="Codec frames per streaming chunk. Audio codebooks are autoregressive — each set depends on the previous — so coherent chunks require enough frames for stable prosody.") - stream_left_context: int = Field(default=25, description="Left context frames for chunk boundary continuity (matches upstream Qwen3-TTS left_context_size=25).") diff --git a/src/server/routes/openai.py b/src/server/routes/openai.py index afabdb9b..2a99448b 100644 --- a/src/server/routes/openai.py +++ b/src/server/routes/openai.py @@ -11,12 +11,19 @@ from fastapi.responses import JSONResponse, StreamingResponse from src.server.deps import _registry, _workers, verify_api_key -from src.server.models.openvino import KokoroLanguage, KokoroVoice -from src.server.models.optimum import PreTrainedTokenizerConfig, RerankerConfig -from src.server.models.ov_genai import OVGenAI_GenConfig, OVGenAI_WhisperGenConfig -from src.server.models.registration import ModelLoadConfig, ModelType, ModelUnloadConfig -from src.server.models.requests_internal import OpenArcBenchRequest -from src.server.models.requests_openai import ( +from src.server.schemas.modeling.contract_kokoro import KokoroLanguage, KokoroVoice +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_qwen3tts import ( + OV_Qwen3TTSCustomVoice, + OV_Qwen3TTSVoiceClone, + OV_Qwen3TTSVoiceDesign, +) +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig +from src.server.schemas.registration import ModelLoadConfig, ModelType, ModelUnloadConfig +from src.server.schemas.requests_internal import OpenArcBenchRequest +from src.server.schemas.requests_openai import ( EmbeddingsRequest, OpenAIChatCompletionRequest, OpenAICompletionRequest, @@ -587,15 +594,30 @@ async def openai_audio_speech(request: OpenAISpeechRequest): ModelType.QWEN3_TTS_VOICE_DESIGN, ModelType.QWEN3_TTS_VOICE_CLONE, ): - if not request.openarc_tts or not request.openarc_tts.qwen3_tts: - raise ValueError("openarc_tts.qwen3_tts required for Qwen3 TTS models") - gen_config = request.openarc_tts.qwen3_tts + if not request.openarc_tts: + raise ValueError("openarc_tts required for Qwen3 TTS models") + _qwen3_tts_field = { + ModelType.QWEN3_TTS_CUSTOM_VOICE: "qwen3_tts_custom_voice", + ModelType.QWEN3_TTS_VOICE_DESIGN: "qwen3_tts_voice_design", + ModelType.QWEN3_TTS_VOICE_CLONE: "qwen3_tts_voice_clone", + }[normalized] + gen_config = getattr(request.openarc_tts, _qwen3_tts_field) + if gen_config is None: + raise ValueError(f"openarc_tts.{_qwen3_tts_field} required for {normalized.value} models") gen_config.input = request.input if request.language is not None and "language" not in gen_config.model_fields_set: gen_config.language = request.language - if request.instructions is not None and "instruct" not in gen_config.model_fields_set: + if ( + request.instructions is not None + and hasattr(gen_config, "instruct") + and "instruct" not in gen_config.model_fields_set + ): gen_config.instruct = request.instructions - if request.voice is not None and "speaker" not in gen_config.model_fields_set: + if ( + request.voice is not None + and isinstance(gen_config, OV_Qwen3TTSCustomVoice) + and "speaker" not in gen_config.model_fields_set + ): gen_config.speaker = request.voice if gen_config.stream: return StreamingResponse( diff --git a/src/server/routes/openarc.py b/src/server/routes/openarc.py index b23d48e9..d31d3b47 100644 --- a/src/server/routes/openarc.py +++ b/src/server/routes/openarc.py @@ -15,10 +15,10 @@ from src.server.deps import _registry, _workers, verify_api_key from src.server.downloader import get_default_models_dir, global_downloader -from src.server.models.ov_genai import OVGenAI_GenConfig -from src.server.models.registration import ModelLoadConfig, ModelUnloadConfig -from src.server.models.requests_internal import OpenArcBenchRequest -from src.server.models.requests_management import ( +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.registration import ModelLoadConfig, ModelUnloadConfig +from src.server.schemas.requests_internal import OpenArcBenchRequest +from src.server.schemas.requests_management import ( DownloaderActionRequest, DownloaderRequest, ) diff --git a/src/server/models/__init__.py b/src/server/schemas/__init__.py similarity index 100% rename from src/server/models/__init__.py rename to src/server/schemas/__init__.py diff --git a/src/server/schemas/modeling/contract_kokoro.py b/src/server/schemas/modeling/contract_kokoro.py new file mode 100644 index 00000000..1c324239 --- /dev/null +++ b/src/server/schemas/modeling/contract_kokoro.py @@ -0,0 +1,130 @@ + +from enum import Enum +from pydantic import BaseModel, Field, field_validator +from typing import Optional + + + +class KokoroLanguage(str, Enum): + """Language codes for Kokoro TTS voices""" + AMERICAN_ENGLISH = "a" + BRITISH_ENGLISH = "b" + JAPANESE = "j" + MANDARIN_CHINESE = "z" + SPANISH = "e" + FRENCH = "f" + HINDI = "h" + ITALIAN = "i" + BRAZILIAN_PORTUGUESE = "p" + +class KokoroVoice(str, Enum): + """Available Kokoro TTS voices organized by language""" + # American English (🇺🇸) - 11F 9M + AF_HEART = "af_heart" + AF_ALLOY = "af_alloy" + AF_AOEDE = "af_aoede" + AF_BELLA = "af_bella" + AF_JESSICA = "af_jessica" + AF_KORE = "af_kore" + AF_NICOLE = "af_nicole" + AF_NOVA = "af_nova" + AF_RIVER = "af_river" + AF_SARAH = "af_sarah" + AF_SKY = "af_sky" + AM_ADAM = "am_adam" + AM_ECHO = "am_echo" + AM_ERIC = "am_eric" + AM_FENRIR = "am_fenrir" + AM_LIAM = "am_liam" + AM_MICHAEL = "am_michael" + AM_ONYX = "am_onyx" + AM_PUCK = "am_puck" + AM_SANTA = "am_santa" + + # British English (🇬🇧) - 4F 4M + BF_ALICE = "bf_alice" + BF_EMMA = "bf_emma" + BF_ISABELLA = "bf_isabella" + BF_LILY = "bf_lily" + BM_DANIEL = "bm_daniel" + BM_FABLE = "bm_fable" + BM_GEORGE = "bm_george" + BM_LEWIS = "bm_lewis" + + # Japanese (🇯🇵) - 4F 1M + JF_ALPHA = "jf_alpha" + JF_GONGITSUNE = "jf_gongitsune" + JF_NEZUMI = "jf_nezumi" + JF_TEBUKURO = "jf_tebukuro" + JM_KUMO = "jm_kumo" + + # Mandarin Chinese (🇨🇳) - 4F 4M + ZF_XIAOBEI = "zf_xiaobei" + ZF_XIAONI = "zf_xiaoni" + ZF_XIAOXIAO = "zf_xiaoxiao" + ZF_XIAOYI = "zf_xiaoyi" + ZM_YUNJIAN = "zm_yunjian" + ZM_YUNXI = "zm_yunxi" + ZM_YUNXIA = "zm_yunxia" + ZM_YUNYANG = "zm_yunyang" + + # Spanish (🇪🇸) - 1F 2M + EF_DORA = "ef_dora" + EM_ALEX = "em_alex" + EM_SANTA = "em_santa" + + # French (🇫🇷) - 1F + FF_SIWIS = "ff_siwis" + + # Hindi (🇮🇳) - 2F 2M + HF_ALPHA = "hf_alpha" + HF_BETA = "hf_beta" + HM_OMEGA = "hm_omega" + HM_PSI = "hm_psi" + + # Italian (🇮🇹) - 1F 1M + IF_SARA = "if_sara" + IM_NICOLA = "im_nicola" + + # Brazilian Portuguese (🇧🇷) - 1F 2M + PF_DORA = "pf_dora" + PM_ALEX = "pm_alex" + PM_SANTA = "pm_santa" + +class OV_KokoroGenConfig(BaseModel): + input: Optional[str] = Field(default=None, description="Injected from top-level request.input by the handler; do not set here.") + voice: KokoroVoice = Field(KokoroVoice.AF_SARAH, description="Voice token from available Kokoro voices") + # Optional weighted blend of voicepacks. Overrides `voice` when set. + # Format: "af_heart,af_nicole" (equal weights) or + # "af_heart:0.7,af_nicole:0.3" (weights normalised by engine). + voice_blend: Optional[str] = Field(af_heart:0.7,af_nicole:0.3 + default=None, + description="Optional weighted blend of voicepacks, e.g. 'af_heart:0.7,af_nicole:0.3'. Overrides `voice`.", + ) + lang_code: KokoroLanguage = Field(KokoroLanguage.AMERICAN_ENGLISH, description="Language code for the voice") + speed: float = Field(1.0, description="Speech speed multiplier") + character_count_chunk: int = Field(100, description="Max characters per chunk") + response_format: str = Field("wav", description="Output format") + + @field_validator("voice_blend") + @classmethod + def _validate_voice_blend(cls, v: Optional[str]) -> Optional[str]: + if v is None or not v.strip(): + return None + parts = [p.strip() for p in v.split(",") if p.strip()] + if not parts: + return None + valid = {item.value for item in KokoroVoice} + for part in parts: + name, _, weight = part.partition(":") + name = name.strip() + if name not in valid: + raise ValueError(f"Unknown voice in blend: {name!r}") + if weight.strip(): + try: + w = float(weight) + except ValueError as exc: + raise ValueError(f"Invalid weight in blend for {name!r}: {weight!r}") from exc + if w < 0: + raise ValueError(f"Negative weight not allowed for {name!r}: {w}") + return v diff --git a/src/server/models/optimum.py b/src/server/schemas/modeling/contract_optimum_emb.py similarity index 87% rename from src/server/models/optimum.py rename to src/server/schemas/modeling/contract_optimum_emb.py index 94c3b70c..6753956a 100644 --- a/src/server/models/optimum.py +++ b/src/server/schemas/modeling/contract_optimum_emb.py @@ -189,47 +189,4 @@ class PreTrainedTokenizerConfig(BaseModel): description=( "Whether or not to print more information and warnings." ) - ) - -class RerankerConfig(BaseModel): - - query: str = Field( - default=None, - description=( - "Phrase to compare documents to." - ) - ) - documents: List[str] = Field( - default=None, - description=( - "Documents to rank." - ) - ) - - prefix: str = Field( - default='<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n', - description=( - "Text to append to start of query. This is model specific." - ) - ) - - suffix: str = Field( - default="<|im_end|>\n<|im_start|>assistant\n\n\n\n\n", - description=( - "Text to append to end of query. This is model specific and configured for Qwen3-Rerank tokenizer." - ) - ) - - instruction: str = Field( - default="Given a search query, retrieve relevant passages that answer the query", - description=( - "Prompt command delivered to the model." - ) - ) - - max_length: int = Field( - default=1024, - description=( - "Maximum sequence length for tokenization." - ) ) \ No newline at end of file diff --git a/src/server/schemas/modeling/contract_optimum_rerank.py b/src/server/schemas/modeling/contract_optimum_rerank.py new file mode 100644 index 00000000..0cabedc0 --- /dev/null +++ b/src/server/schemas/modeling/contract_optimum_rerank.py @@ -0,0 +1,48 @@ + +from typing import List, Union +from pydantic import BaseModel, Field + + + +class RerankerConfig(BaseModel): + + query: str = Field( + default=None, + description=( + "Phrase to compare documents to." + ) + ) + documents: List[str] = Field( + default=None, + description=( + "Documents to rank." + ) + ) + + prefix: str = Field( + default='<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n', + description=( + "Text to append to start of query. This is model specific." + ) + ) + + suffix: str = Field( + default="<|im_end|>\n<|im_start|>assistant\n\n\n\n\n", + description=( + "Text to append to end of query. This is model specific and configured for Qwen3-Rerank tokenizer." + ) + ) + + instruction: str = Field( + default="Given a search query, retrieve relevant passages that answer the query", + description=( + "Prompt command delivered to the model." + ) + ) + + max_length: int = Field( + default=1024, + description=( + "Maximum sequence length for tokenization." + ) + ) \ No newline at end of file diff --git a/src/server/models/ov_genai.py b/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py similarity index 96% rename from src/server/models/ov_genai.py rename to src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py index d9249021..8216fa93 100644 --- a/src/server/models/ov_genai.py +++ b/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py @@ -1,7 +1,5 @@ from typing import Any, Dict, List, Optional - from pydantic import BaseModel, Field - from src.server.utils.chat import flatten_messages class OVGenAI_GenConfig(BaseModel): @@ -93,6 +91,3 @@ def text_messages(self) -> List[Dict[str, Any]]: """Messages with their `content` coerced to plain strings for text models.""" return flatten_messages(self.messages) - -class OVGenAI_WhisperGenConfig(BaseModel): - audio_base64: str = Field(..., description="Base64 encoded audio") diff --git a/src/server/schemas/modeling/contract_qwen3asr.py b/src/server/schemas/modeling/contract_qwen3asr.py new file mode 100644 index 00000000..81c4d507 --- /dev/null +++ b/src/server/schemas/modeling/contract_qwen3asr.py @@ -0,0 +1,28 @@ +from enum import Enum +from pydantic import BaseModel, Field, field_validator +from typing import Optional + + + + +class OV_Qwen3ASRGenConfig(BaseModel): + audio_base64: str | None = Field(default=None, description="Base64 encoded audio payload (injected from file when omitted)") + language: Optional[str] = Field(default=None, description="Optional forced language") + max_tokens: int = Field(default=1024, description="Maximum generated tokens per chunk") + max_chunk_sec: float = Field(default=30.0, description="Chunk size upper bound in seconds") + search_expand_sec: float = Field(default=5.0, description="Boundary search expansion in seconds") + min_window_ms: float = Field(default=100.0, description="Energy window in milliseconds") + + @field_validator("max_tokens") + @classmethod + def _validate_max_tokens(cls, v: int) -> int: + if v <= 0: + raise ValueError("max_tokens must be positive") + return v + + @field_validator("max_chunk_sec", "search_expand_sec", "min_window_ms") + @classmethod + def _validate_positive_float(cls, v: float) -> float: + if v <= 0: + raise ValueError("numeric values must be positive") + return v \ No newline at end of file diff --git a/src/server/schemas/modeling/contract_qwen3tts.py b/src/server/schemas/modeling/contract_qwen3tts.py new file mode 100644 index 00000000..aa76cbe0 --- /dev/null +++ b/src/server/schemas/modeling/contract_qwen3tts.py @@ -0,0 +1,64 @@ +from enum import Enum +from pydantic import BaseModel, Field, field_validator +from typing import Optional + + + +class OV_Qwen3TTSGenConfig(BaseModel): + """Base config for all OVQwen3TTS request parameters: shared sampling + streaming + transport + the injected `input`/`language` shared by every mode. + + The loaded model's `model_type` (registration.py) determines which mode subclass the + engine runs; supply the subclass matching that mode: + + - qwen3_tts_custom_voice (OV_Qwen3TTSCustomVoice) : input, speaker, language, instruct + - qwen3_tts_voice_design (OV_Qwen3TTSVoiceDesign) : input, voice_description, language + - qwen3_tts_voice_clone (OV_Qwen3TTSVoiceClone) : input, ref_audio_b64, ref_text, x_vector_only, language, instruct + + All modes inherit the sampling + streaming fields below. + """ + # --- shared content (all modes) --- + input: Optional[str] = Field(default=None, description="Injected from top-level request.input by the handler; do not set here.") + language: str | None = Field(default=None, description="[all] Force output language. None = auto-detect.") + # --- sampling (all modes) --- + max_new_tokens: int = Field(default=2048, description="Maximum codec frames to generate.") + do_sample: bool = Field(default=True, description="Sample from logits. False = greedy.") + top_k: int = Field(default=50, description="Top-k filter for talker logits.") + top_p: float = Field(default=1.0, description="Nucleus filter for talker logits. 1.0 = off.") + temperature: float = Field(default=0.9, description="Temperature scaling for talker logits.") + repetition_penalty: float = Field(default=1.05, description="Repetition penalty on first-codebook history. 1.0 = off.") + non_streaming_mode: bool = Field(default=True, description="True = all text tokens in prefill; False = drip-fed during decode.") + subtalker_do_sample: bool = Field(default=True, description="Sample sub-codebook logits.") + subtalker_top_k: int = Field(default=50, description="Top-k for code predictor.") + subtalker_top_p: float = Field(default=1.0, description="Nucleus filter for code predictor.") + subtalker_temperature: float = Field(default=0.9, description="Temperature for code predictor.") + # --- streaming (HTTP: audio/L16 chunked response when stream=True) --- + + # defaults taken from https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py#L886 + # these apply only for the 12.5hz tokenizer model. + stream: bool = Field(default=True, description="Enable streaming audio output (chunked PCM).") + stream_chunk_frames: int = Field(default=300, description="Codec frames per streaming chunk. Audio codebooks are autoregressive — each set depends on the previous — so coherent chunks require enough frames for stable prosody.") + stream_left_context: int = Field(default=25, description="Left context frames for chunk boundary continuity (matches upstream Qwen3-TTS left_context_size=25).") + + +class OV_Qwen3TTSCustomVoice(OV_Qwen3TTSGenConfig): + """qwen3_tts_custom_voice mode: synthesize with a predefined speaker name.""" + speaker: str | None = Field(default=None, description="[custom_voice] Predefined speaker name.") + instruct: str | None = Field(default=None, description="[custom_voice] Optional style instruction.") + + +class OV_Qwen3TTSVoiceDesign(OV_Qwen3TTSGenConfig): + """qwen3_tts_voice_design mode: synthesize from a free-form voice description. + + Note: `voice_description` is fed into the engine's `instruct` slot internally; this + config does not expose `instruct`. + """ + voice_description: str | None = Field(default=None, description="[voice_design] Free-form voice description.") + + +class OV_Qwen3TTSVoiceClone(OV_Qwen3TTSGenConfig): + """qwen3_tts_voice_clone mode: clone a reference audio's voice.""" + ref_audio_b64: str | None = Field(default=None, description="[voice_clone] Base64-encoded reference WAV.") + ref_text: str | None = Field(default=None, description="[voice_clone] Transcript of reference audio (enables ICL).") + x_vector_only: bool = Field(default=False, description="[voice_clone] Use x-vector embedding only; skip ICL even if ref_text is set.") + instruct: str | None = Field(default=None, description="[voice_clone] Optional style instruction.") diff --git a/src/server/schemas/modeling/contract_whisper.py b/src/server/schemas/modeling/contract_whisper.py new file mode 100644 index 00000000..0618bc21 --- /dev/null +++ b/src/server/schemas/modeling/contract_whisper.py @@ -0,0 +1,5 @@ +from typing import Any, Dict, List, Optional +from pydantic import BaseModel, Field + +class OVGenAI_WhisperGenConfig(BaseModel): + audio_base64: str = Field(..., description="Base64 encoded audio") diff --git a/src/server/models/registration.py b/src/server/schemas/registration.py similarity index 100% rename from src/server/models/registration.py rename to src/server/schemas/registration.py diff --git a/src/server/models/requests_internal.py b/src/server/schemas/requests_internal.py similarity index 100% rename from src/server/models/requests_internal.py rename to src/server/schemas/requests_internal.py diff --git a/src/server/models/requests_management.py b/src/server/schemas/requests_management.py similarity index 100% rename from src/server/models/requests_management.py rename to src/server/schemas/requests_management.py diff --git a/src/server/models/requests_openai.py b/src/server/schemas/requests_openai.py similarity index 78% rename from src/server/models/requests_openai.py rename to src/server/schemas/requests_openai.py index f769ee31..14142e22 100644 --- a/src/server/models/requests_openai.py +++ b/src/server/schemas/requests_openai.py @@ -2,12 +2,14 @@ from pydantic import BaseModel -from src.server.models.openvino import ( - OV_KokoroGenConfig, - OV_Qwen3ASRGenConfig, - OV_Qwen3TTSGenConfig, +from src.server.schemas.modeling.contract_kokoro import OV_KokoroGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_qwen3tts import ( + OV_Qwen3TTSCustomVoice, + OV_Qwen3TTSVoiceClone, + OV_Qwen3TTSVoiceDesign, ) -from src.server.models.optimum import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig class OpenArcASRConfig(BaseModel): @@ -17,7 +19,11 @@ class OpenArcASRConfig(BaseModel): class OpenArcTTSConfig(BaseModel): kokoro: Optional[OV_KokoroGenConfig] = None - qwen3_tts: Optional[OV_Qwen3TTSGenConfig] = None + # Request shape is locked at model load time by the loaded model's model_type: + # exactly one of the three qwen3_tts_* fields is valid, matching the loaded mode. + qwen3_tts_custom_voice: Optional[OV_Qwen3TTSCustomVoice] = None + qwen3_tts_voice_design: Optional[OV_Qwen3TTSVoiceDesign] = None + qwen3_tts_voice_clone: Optional[OV_Qwen3TTSVoiceClone] = None class OpenAIChatCompletionRequest(BaseModel): diff --git a/src/server/worker_registry.py b/src/server/worker_registry.py index 0dff452e..739fe01b 100644 --- a/src/server/worker_registry.py +++ b/src/server/worker_registry.py @@ -18,11 +18,15 @@ from src.engine.optimum.optimum_emb import Optimum_EMB from src.engine.optimum.optimum_rr import Optimum_RR -from src.server.models.openvino import OV_KokoroGenConfig, OV_Qwen3ASRGenConfig, OV_Qwen3TTSGenConfig -from src.server.models.ov_genai import OVGenAI_GenConfig, OVGenAI_WhisperGenConfig -from src.server.models.optimum import PreTrainedTokenizerConfig, RerankerConfig +from src.server.schemas.modeling.contract_kokoro import OV_KokoroGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_qwen3tts import OV_Qwen3TTSGenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig from src.server.model_registry import ModelRecord, ModelRegistry -from src.server.models.registration import ModelType +from src.server.schemas.registration import ModelType logger = logging.getLogger(__name__) diff --git a/gpu-metrics/gpu_metrics.cpp b/src/utilities/gpu_metrics/gpu_metrics.cpp similarity index 100% rename from gpu-metrics/gpu_metrics.cpp rename to src/utilities/gpu_metrics/gpu_metrics.cpp diff --git a/gpu-metrics/pyproject.toml b/src/utilities/gpu_metrics/pyproject.toml similarity index 100% rename from gpu-metrics/pyproject.toml rename to src/utilities/gpu_metrics/pyproject.toml diff --git a/gpu-metrics/setup.py b/src/utilities/gpu_metrics/setup.py similarity index 100% rename from gpu-metrics/setup.py rename to src/utilities/gpu_metrics/setup.py diff --git a/test_gpu_metrics.py b/src/utilities/gpu_metrics/test_gpu_metrics.py similarity index 99% rename from test_gpu_metrics.py rename to src/utilities/gpu_metrics/test_gpu_metrics.py index 0244a6b6..4ad835ea 100644 --- a/test_gpu_metrics.py +++ b/src/utilities/gpu_metrics/test_gpu_metrics.py @@ -1,6 +1,8 @@ import sys import json + + try: import gpu_metrics except ImportError as e: diff --git a/tests/integration/test_optimum_emb_integration.py b/tests/integration/test_optimum_emb_integration.py index 7bffdd46..ab19a459 100644 --- a/tests/integration/test_optimum_emb_integration.py +++ b/tests/integration/test_optimum_emb_integration.py @@ -4,8 +4,8 @@ from test_model_path import model_path from src.engine.optimum.optimum_emb import Optimum_EMB -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.optimum import PreTrainedTokenizerConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig MODEL_PATH = model_path("Qwen3-Embedding-0.6B-int8_asym-ov") diff --git a/tests/integration/test_optimum_rr_integration.py b/tests/integration/test_optimum_rr_integration.py index 00160cc0..bc118112 100644 --- a/tests/integration/test_optimum_rr_integration.py +++ b/tests/integration/test_optimum_rr_integration.py @@ -4,8 +4,8 @@ from test_model_path import model_path from src.engine.optimum.optimum_rr import Optimum_RR -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.optimum import RerankerConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig MODEL_PATH = model_path("Qwen3-Reranker-0.6B-fp16-ov") diff --git a/tests/integration/test_ov_genai_kokoro_integration.py b/tests/integration/test_ov_genai_kokoro_integration.py index a0fe2197..a9327366 100644 --- a/tests/integration/test_ov_genai_kokoro_integration.py +++ b/tests/integration/test_ov_genai_kokoro_integration.py @@ -5,8 +5,8 @@ from test_model_path import model_path from src.engine.openvino.kokoro import OV_Kokoro -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.openvino import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_kokoro import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig MODEL_PATH = model_path("Kokoro-82M-FP16-OpenVINO") diff --git a/tests/integration/test_ov_genai_llm_integration.py b/tests/integration/test_ov_genai_llm_integration.py index ed3f8241..218a695a 100644 --- a/tests/integration/test_ov_genai_llm_integration.py +++ b/tests/integration/test_ov_genai_llm_integration.py @@ -2,8 +2,8 @@ from test_model_path import model_path from src.engine.ov_genai.llm import OVGenAI_LLM -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig MODEL_PATH = model_path("Qwen3-0.6B-int8-ov") diff --git a/tests/integration/test_ov_genai_qwen3_asr_integration.py b/tests/integration/test_ov_genai_qwen3_asr_integration.py index c20d80ed..03853697 100644 --- a/tests/integration/test_ov_genai_qwen3_asr_integration.py +++ b/tests/integration/test_ov_genai_qwen3_asr_integration.py @@ -6,10 +6,10 @@ import pytest # type: ignore[import] -from src.server.models.openvino import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig from src.engine.openvino.qwen3_asr.qwen3_asr import OVQwen3ASR from test_model_path import model_path -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType diff --git a/tests/integration/test_ov_genai_vlm_integration.py b/tests/integration/test_ov_genai_vlm_integration.py index 1eb88b17..3a72916c 100644 --- a/tests/integration/test_ov_genai_vlm_integration.py +++ b/tests/integration/test_ov_genai_vlm_integration.py @@ -8,8 +8,8 @@ from test_model_path import model_path from src.engine.ov_genai.vlm import OVGenAI_VLM -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig MODEL_PATH = model_path("Qwen2.5-VL-3B-Instruct-int4_sym-ov") diff --git a/tests/integration/test_ov_genai_whisper_integration.py b/tests/integration/test_ov_genai_whisper_integration.py index c49ea96c..08067d3b 100644 --- a/tests/integration/test_ov_genai_whisper_integration.py +++ b/tests/integration/test_ov_genai_whisper_integration.py @@ -7,8 +7,8 @@ from test_model_path import model_path from src.engine.ov_genai.whisper import OVGenAI_Whisper -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_WhisperGenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig MODEL_PATH = model_path("whisper-tiny-int8-ov") diff --git a/tests/unit/test_model_registry_integration.py b/tests/unit/test_model_registry_integration.py index a6122ba1..d4daee3d 100644 --- a/tests/unit/test_model_registry_integration.py +++ b/tests/unit/test_model_registry_integration.py @@ -4,7 +4,7 @@ import src.server.model_registry as model_registry_module from src.server.model_registry import ModelRegistry -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType class _DummyModel: diff --git a/tests/unit/test_model_registry_unit.py b/tests/unit/test_model_registry_unit.py index 9dfb99e3..121ddcfe 100644 --- a/tests/unit/test_model_registry_unit.py +++ b/tests/unit/test_model_registry_unit.py @@ -5,7 +5,7 @@ import src.server.model_registry as registry_module from src.server.model_registry import ModelRegistry, create_model_instance -from src.server.models.registration import ( +from src.server.schemas.registration import ( EngineType, ModelLoadConfig, ModelStatus, diff --git a/tests/unit/test_openai_audio_transcriptions_unit.py b/tests/unit/test_openai_audio_transcriptions_unit.py index 6fae1f47..5df32c31 100644 --- a/tests/unit/test_openai_audio_transcriptions_unit.py +++ b/tests/unit/test_openai_audio_transcriptions_unit.py @@ -6,7 +6,7 @@ import pytest # type: ignore[import] import src.server.routes.openai as openai_module -from src.server.models.registration import ModelType +from src.server.schemas.registration import ModelType _AUDIO_BYTES = b"audio-bytes" diff --git a/tests/unit/test_optimum_emb_unit.py b/tests/unit/test_optimum_emb_unit.py index 2a69da48..35a6fcc7 100644 --- a/tests/unit/test_optimum_emb_unit.py +++ b/tests/unit/test_optimum_emb_unit.py @@ -7,8 +7,8 @@ import src.engine.optimum.optimum_emb as emb_module from src.engine.optimum.optimum_emb import Optimum_EMB -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.optimum import PreTrainedTokenizerConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig MODEL_PATH = "/mnt/Ironwolf-4TB/Models/Pytorch/Qwen/Qwen3-Embed-0.6B-INT8-ASYM-ov" diff --git a/tests/unit/test_optimum_rr_unit.py b/tests/unit/test_optimum_rr_unit.py index 8fb0d716..d0476791 100644 --- a/tests/unit/test_optimum_rr_unit.py +++ b/tests/unit/test_optimum_rr_unit.py @@ -7,8 +7,8 @@ import src.engine.optimum.optimum_rr as rr_module from src.engine.optimum.optimum_rr import Optimum_RR -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.optimum import RerankerConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig MODEL_PATH = "/mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Qwen3-Reranker-0.6B-fp16-ov" diff --git a/tests/unit/test_ov_genai_kokoro_unit.py b/tests/unit/test_ov_genai_kokoro_unit.py index bc746588..e3c1803c 100644 --- a/tests/unit/test_ov_genai_kokoro_unit.py +++ b/tests/unit/test_ov_genai_kokoro_unit.py @@ -7,8 +7,8 @@ import src.engine.openvino.kokoro as kokoro_module from src.engine.openvino.kokoro import OV_Kokoro -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.openvino import ( +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_kokoro import ( KokoroLanguage, KokoroVoice, OV_KokoroGenConfig, diff --git a/tests/unit/test_ov_genai_llm_unit.py b/tests/unit/test_ov_genai_llm_unit.py index 54c9347c..91433e72 100644 --- a/tests/unit/test_ov_genai_llm_unit.py +++ b/tests/unit/test_ov_genai_llm_unit.py @@ -4,8 +4,8 @@ import src.engine.ov_genai.llm as llm_module from src.engine.ov_genai.llm import OVGenAI_LLM -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig MODEL_PATH ="some_fake_url/Qwen3-Reranker-0.6B-fp16-ov" diff --git a/tests/unit/test_ov_genai_vlm_unit.py b/tests/unit/test_ov_genai_vlm_unit.py index 9c5ac9ae..870a9617 100644 --- a/tests/unit/test_ov_genai_vlm_unit.py +++ b/tests/unit/test_ov_genai_vlm_unit.py @@ -7,8 +7,8 @@ import src.engine.ov_genai.vlm as vlm_module from src.engine.ov_genai.vlm import OVGenAI_VLM -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_GenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig MODEL_PATH = "/mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Qwen2.5-VL-3B-Instruct-int4_sym-ov" diff --git a/tests/unit/test_ov_genai_whisper_unit.py b/tests/unit/test_ov_genai_whisper_unit.py index d32611e2..ff2d2f41 100644 --- a/tests/unit/test_ov_genai_whisper_unit.py +++ b/tests/unit/test_ov_genai_whisper_unit.py @@ -7,8 +7,8 @@ import src.engine.ov_genai.whisper as whisper_module from src.engine.ov_genai.whisper import OVGenAI_Whisper -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.ov_genai import OVGenAI_WhisperGenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig MODEL_PATH = "/mnt/Ironwolf-4TB/Models/OpenVINO/Whisper/distil-whisper-large-v3-int8-ov" diff --git a/tests/unit/test_qwen3_asr_unit.py b/tests/unit/test_qwen3_asr_unit.py index 76128394..9e883681 100644 --- a/tests/unit/test_qwen3_asr_unit.py +++ b/tests/unit/test_qwen3_asr_unit.py @@ -6,8 +6,8 @@ import src.engine.openvino.qwen3_asr.qwen3_asr as qwen3_asr_module from src.engine.openvino.qwen3_asr.qwen3_asr import OVQwen3ASR, SAMPLE_RATE -from src.server.models.openvino import OV_Qwen3ASRGenConfig -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType from src.engine.openvino.qwen3_asr.qwen3_asr_utils import ( LANGUAGE_CODE_TO_NAME, SUPPORTED_LANGUAGES, diff --git a/tests/unit/test_readiness_unit.py b/tests/unit/test_readiness_unit.py index f905bde0..0adf8b2c 100644 --- a/tests/unit/test_readiness_unit.py +++ b/tests/unit/test_readiness_unit.py @@ -5,7 +5,7 @@ import src.server.model_registry as registry_module from src.server.model_registry import ModelRecord, ModelRegistry -from src.server.models.registration import ( +from src.server.schemas.registration import ( EngineType, ModelLoadConfig, ModelStatus, diff --git a/tests/unit/test_tool_call_parser_unit.py b/tests/unit/test_tool_call_parser_unit.py index 774320e6..9d4e12a7 100644 --- a/tests/unit/test_tool_call_parser_unit.py +++ b/tests/unit/test_tool_call_parser_unit.py @@ -5,7 +5,7 @@ from fastapi.responses import StreamingResponse import src.server.routes.openai as openai_routes -from src.server.models.requests_openai import OpenAIChatCompletionRequest +from src.server.schemas.requests_openai import OpenAIChatCompletionRequest class _DummyRequest: diff --git a/tests/unit/test_worker_registry_integration.py b/tests/unit/test_worker_registry_integration.py index 7a35c222..32cc2549 100644 --- a/tests/unit/test_worker_registry_integration.py +++ b/tests/unit/test_worker_registry_integration.py @@ -5,10 +5,13 @@ import src.server.model_registry as model_registry_module import src.server.worker_registry as worker_module from src.server.model_registry import ModelRegistry -from src.server.models.registration import EngineType, ModelLoadConfig, ModelType -from src.server.models.openvino import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig, OV_Qwen3ASRGenConfig -from src.server.models.optimum import PreTrainedTokenizerConfig, RerankerConfig -from src.server.models.ov_genai import OVGenAI_GenConfig, OVGenAI_WhisperGenConfig +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_kokoro import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig def _make_worker(response_value, metrics_value, supports_stream: bool = False, segments_value=None): diff --git a/tests/unit/test_worker_registry_unit.py b/tests/unit/test_worker_registry_unit.py index 7ec84396..2019cb22 100644 --- a/tests/unit/test_worker_registry_unit.py +++ b/tests/unit/test_worker_registry_unit.py @@ -4,10 +4,13 @@ import src.server.worker_registry as worker_module from src.server.model_registry import ModelRecord, ModelRegistry -from src.server.models.registration import ModelType -from src.server.models.openvino import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig, OV_Qwen3ASRGenConfig -from src.server.models.optimum import PreTrainedTokenizerConfig, RerankerConfig -from src.server.models.ov_genai import OVGenAI_GenConfig, OVGenAI_WhisperGenConfig +from src.server.schemas.registration import ModelType +from src.server.schemas.modeling.contract_kokoro import KokoroLanguage, KokoroVoice, OV_KokoroGenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig +from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_whisper import OVGenAI_WhisperGenConfig def _make_worker(response_value, metrics_value, supports_stream: bool = False, segments_value=None):