From 77170bdcfeb2be15431d008291adbbb7e5fba9c1 Mon Sep 17 00:00:00 2001 From: Rui Ren Date: Tue, 14 Jul 2026 10:45:40 -0700 Subject: [PATCH 1/2] add file based support for streaming model --- .../generative/audio/audio_session.cc | 193 ++++++++++++++++++ .../generative/audio/audio_session.h | 10 + 2 files changed, 203 insertions(+) diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc index 06a597ea9..5b2f1cac2 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc @@ -73,6 +73,41 @@ std::string JoinTokens(const std::vector& token_texts) { } // namespace +namespace { + const std::unordered_map NemotronLangIdMap = { + ["en"] = "0", ["en-US"] = "0", ["en-GB"] = "1", + ["es-ES"] = "2", ["es"] = "3", ["es-US"] = "3", + ["zh-CN"] = "4", + ["hi"] = "6", ["hi-IN"] = "6", ["ar"] = "7", ["ar-AR"] = "7", + ["fr"] = "8", ["fr-FR"] = "8", ["fr-CA"] = "100", + ["de"] = "9", ["de-DE"] = "9", + ["ja"] = "10", ["ja-JP"] = "10", ["ru"] = "11", ["ru-RU"] = "11", + ["pt-BR"] = "12", ["pt"] = "13", ["pt-PT"] = "13", + ["ko"] = "14", ["ko-KR"] = "14", ["it"] = "15", ["it-IT"] = "15", + ["nl"] = "16", ["nl-NL"] = "16", ["pl"] = "17", ["pl-PL"] = "17", + ["tr"] = "18", ["tr-TR"] = "18", ["uk"] = "19", ["uk-UA"] = "19", + ["ro"] = "20", ["ro-RO"] = "20", + ["el"] = "21", ["el-GR"] = "21", ["cs"] = "22", ["cs-CZ"] = "22", + ["hu"] = "23", ["hu-HU"] = "23", ["sv"] = "24", ["sv-SE"] = "24", + ["da"] = "25", ["da-DK"] = "25", + ["fi"] = "26", ["fi-FI"] = "26", ["sk"] = "28", ["sk-SK"] = "28", + ["hr"] = "29", ["hr-HR"] = "29", ["bg"] = "30", ["bg-BG"] = "30", + ["lt"] = "31", ["lt-LT"] = "31", ["th"] = "32", ["th-TH"] = "32", + ["vi"] = "33", ["vi-VN"] = "33", + ["et"] = "60", ["et-EE"] = "60", ["lv"] = "61", ["lv-LV"] = "61", + ["sl"] = "62", ["sl-SI"] = "62", ["he"] = "64", ["he-IL"] = "64", + ["auto"] = "101", + ["mt"] = "102", ["mt-MT"] = "102", ["nb"] = "103", ["nb-NO"] = "103", + ["nn"] = "104", ["nn-NO"] = "104", + }; + + std::string ToLowerAscii(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); + return s; + } + } // namespace fl + + AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { @@ -422,6 +457,13 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, fmt::format("Audio file not found: '{}'", req.filename)); } + // If the model is a Nemotron speech model, process the transcription using the Nemotron-specific method. + // e.g. RNNT models require Nemotron-specific transcription handling. + if (IsNemotronSpeechModel()) { + ProcessNemotronFileTranscription(req, original_request, response); + return; + } + // Build generation options from session defaults SearchOptions options = session_options_; @@ -502,4 +544,155 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json total_tokens, prompt_tokens, completion_tokens)); } + +bool AudioSession::IsNemotronSpeechModel() const { + const auto& cfg = Model().GetConfig(); + return cfg.model.has_value() && cfg.model->type == "nemotron_speech"; +} + + +void AudioSession::TryNemotronLanguageId(OgaGenerator& generator, + const std::string& language) const { + if (language.empty()) { + return; + } + + const auto key = ToLowerAscii(language); + auto it = NemotronLanguageIdMap().find(key); + if (it == NemotronLanguageIdMap().end()) { + return; + } + + + try { + generator.SetRuntimeOption("lang_id", it->second.c_str()); + } catch (const std::exception& e) { + logger_.Log(LogLevel::Error, fmt::format("Failed to set Nemotron language ID: {}", e.what())); + } + } + + + void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, + const Requests& original_request, + const Response& response) { + + constexpr size_t kInitialTokenCapacity = 256; + + if (original_request.canceled) { + response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE; + return; + } + + const float temperature = req.temperature.value_or( + session_options_.temperature.value_or(0.0f) + ); + + const std::string language = req.language.value_or(""); + + std::vector samples = LoadPcmWavAsFloatSamples(req.audio_file_path); + + auto& oga_model = Model().GetOgaModel(); + + auto processor = OgaStreamingProcessor::Create(oga_model); + + auto tokenizer_stream = OgaTokenizerStream::Create(Model().GetOgaModel()); + + std::vector token_texts; + + token_texts.reserve(kInitialTokenCapacity); + + std::vector> segments; + + segments.reserve(kInitialSegmentCapacity); + + int completion_tokens = 0; + + auto callback = CreateCallbackHandler(original_request); + + // Process initial tensors with fresh generator + + if (!original_request.canceled) { + // Process initial tensors here + + auto tensors = processor->Process(samples.data(), samples.size()); + + if (tensors) { + auto gen_params = OgaGeneratorParams::Create(oga_model); + + gen_params->SetSearchOption("temperature", temperature); + + auto generator = OgaGenerator::Create(oga_model, *gen_params); + + TryNemotronLanguageId(*generator, language); + + generator->SetInputs(*tensors); + + DecodeTokens(*generator, *tokenizer_stream, token_texts, segments, + callback, original_request, completion_tokens); + } + } + + // Flush tensors with fresh generator (Important: avoid done-state reuse bug) + + if (!original_request.canceled) { + auto flush_tensors = processor->Flush(); + + if (flush_tensors) { + + auto gen_params = OgaGeneratorParams::Create(oga_model); + + gen_params->SetSearchOption("temperature", temperature); + + auto generator = OgaGenerator::Create(oga_model, *gen_params); + + TryNemotronLanguageId(*generator, language); + + generator->SetInputs(*flush_tensors); + + DecodeTokens(*generator, *tokenizer_stream, token_texts, segments, + callback, original_request, completion_tokens); + } + } + + std::string full_text = JoinTokens(token_texts); + response.items.push_back(BuildSpeechResult(std::move(full_text), std::move(segments))); + + response.finish_reason = original_request.canceled ? FOUNDRY_FINISH_REASON_CANCELED : FOUNDRY_FINISH_REASON_COMPLETED; + + response.usage.prompt_tokens = 0; + + response.usage.completion_tokens = completion_tokens; + + response.usage.total_tokens = response.usage.prompt_tokens + response.usage.completion_tokens; + } + + std::vector AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) { + + std::ifstream in(audio_file_path, std::ios::binary); + + if (!in) { + FL_THROW("Failed to open audio file: " + audio_file_path); + } + + // TODO: + // 1) Read RIFF/WAVE header, fail if invalid. + // 2) Iterate chunks safely: + // - read chunk id + size + // - bounds-check size + // - if "fmt " and size < 16 => throw (GitOps feedback fix) + // - parse format/channels/sampleRate/bitsPerSample + // - capture "data" bytes + // 3) Enforce sampleRate == 16000 + // 4) Convert: + // - PCM16 -> float [-1,1], average channels to mono if needed + // - (optional parity) float32 PCM -> mono float + // 5) return std::vector + + FL_THROW(FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED, + "LoadPcmWavAsFloatSamples not implemented yet"); +} + + } + + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h index 1a35d096a..1dd913f81 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h @@ -53,6 +53,16 @@ class AudioSession : public Session { void ProcessAudioTranscriptionJson(const std::string& request_json, const Request& original_request, Response& response); + bool IsNemotronSpeechModel() const; + + void ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, + const Request& original_request, + Response& response); + + void TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const; + + static std::vector LoadPcmWavAsFloatSample(const std::string&& audio_file_path); + /// Process a streaming audio request: an AudioItem (format descriptor) + an ItemQueue (PCM chunks). void ProcessStreamingAudio(const AudioItem& format_item, ItemQueue& queue, const Request& request, Response& response); From d5eafd66118dced5b244537af273f9cc477dcab2 Mon Sep 17 00:00:00 2001 From: rui-ren Date: Wed, 15 Jul 2026 10:06:12 -0700 Subject: [PATCH 2/2] update the cpp code for streaming model --- .../generative/audio/audio_session.cc | 405 ++++++++++++------ .../generative/audio/audio_session.h | 3 +- 2 files changed, 271 insertions(+), 137 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc index 5b2f1cac2..500c982e6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc @@ -19,9 +19,15 @@ #include "utils.h" #include +#include +#include +#include +#include +#include #include #include #include +#include namespace fl { @@ -71,41 +77,45 @@ std::string JoinTokens(const std::vector& token_texts) { return out; } -} // namespace +const std::unordered_map& NemotronLanguageIdMap() { + static const std::unordered_map kMap = { + {"en", "0"}, {"en-us", "0"}, {"en-gb", "1"}, {"es-es", "2"}, {"es", "3"}, + {"es-us", "3"}, {"zh-cn", "4"}, {"hi", "6"}, {"hi-in", "6"}, {"ar", "7"}, + {"ar-ar", "7"}, {"fr", "8"}, {"fr-fr", "8"}, {"fr-ca", "100"}, {"de", "9"}, + {"de-de", "9"}, {"ja", "10"}, {"ja-jp", "10"}, {"ru", "11"}, {"ru-ru", "11"}, + {"pt-br", "12"}, {"pt", "13"}, {"pt-pt", "13"}, {"ko", "14"}, {"ko-kr", "14"}, + {"it", "15"}, {"it-it", "15"}, {"nl", "16"}, {"nl-nl", "16"}, {"pl", "17"}, + {"pl-pl", "17"}, {"tr", "18"}, {"tr-tr", "18"}, {"uk", "19"}, {"uk-ua", "19"}, + {"ro", "20"}, {"ro-ro", "20"}, {"el", "21"}, {"el-gr", "21"}, {"cs", "22"}, + {"cs-cz", "22"}, {"hu", "23"}, {"hu-hu", "23"}, {"sv", "24"}, {"sv-se", "24"}, + {"da", "25"}, {"da-dk", "25"}, {"fi", "26"}, {"fi-fi", "26"}, {"sk", "28"}, + {"sk-sk", "28"}, {"hr", "29"}, {"hr-hr", "29"}, {"bg", "30"}, {"bg-bg", "30"}, + {"lt", "31"}, {"lt-lt", "31"}, {"th", "32"}, {"th-th", "32"}, {"vi", "33"}, + {"vi-vn", "33"}, {"et", "60"}, {"et-ee", "60"}, {"lv", "61"}, {"lv-lv", "61"}, + {"sl", "62"}, {"sl-si", "62"}, {"he", "64"}, {"he-il", "64"}, {"auto", "101"}, + {"mt", "102"}, {"mt-mt", "102"}, {"nb", "103"}, {"nb-no", "103"}, {"nn", "104"}, + {"nn-no", "104"}, + }; + return kMap; +} -namespace { - const std::unordered_map NemotronLangIdMap = { - ["en"] = "0", ["en-US"] = "0", ["en-GB"] = "1", - ["es-ES"] = "2", ["es"] = "3", ["es-US"] = "3", - ["zh-CN"] = "4", - ["hi"] = "6", ["hi-IN"] = "6", ["ar"] = "7", ["ar-AR"] = "7", - ["fr"] = "8", ["fr-FR"] = "8", ["fr-CA"] = "100", - ["de"] = "9", ["de-DE"] = "9", - ["ja"] = "10", ["ja-JP"] = "10", ["ru"] = "11", ["ru-RU"] = "11", - ["pt-BR"] = "12", ["pt"] = "13", ["pt-PT"] = "13", - ["ko"] = "14", ["ko-KR"] = "14", ["it"] = "15", ["it-IT"] = "15", - ["nl"] = "16", ["nl-NL"] = "16", ["pl"] = "17", ["pl-PL"] = "17", - ["tr"] = "18", ["tr-TR"] = "18", ["uk"] = "19", ["uk-UA"] = "19", - ["ro"] = "20", ["ro-RO"] = "20", - ["el"] = "21", ["el-GR"] = "21", ["cs"] = "22", ["cs-CZ"] = "22", - ["hu"] = "23", ["hu-HU"] = "23", ["sv"] = "24", ["sv-SE"] = "24", - ["da"] = "25", ["da-DK"] = "25", - ["fi"] = "26", ["fi-FI"] = "26", ["sk"] = "28", ["sk-SK"] = "28", - ["hr"] = "29", ["hr-HR"] = "29", ["bg"] = "30", ["bg-BG"] = "30", - ["lt"] = "31", ["lt-LT"] = "31", ["th"] = "32", ["th-TH"] = "32", - ["vi"] = "33", ["vi-VN"] = "33", - ["et"] = "60", ["et-EE"] = "60", ["lv"] = "61", ["lv-LV"] = "61", - ["sl"] = "62", ["sl-SI"] = "62", ["he"] = "64", ["he-IL"] = "64", - ["auto"] = "101", - ["mt"] = "102", ["mt-MT"] = "102", ["nb"] = "103", ["nb-NO"] = "103", - ["nn"] = "104", ["nn-NO"] = "104", - }; - - std::string ToLowerAscii(std::string s) { - std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); - return s; - } - } // namespace fl +std::string ToLowerAscii(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return s; +} + +bool IsLanguageToken(const std::string& token) { + size_t start = token.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) { + return false; + } + + size_t end = token.find_last_not_of(" \t\r\n"); + return end >= start && token[start] == '<' && token[end] == '>'; +} + +} // namespace AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, @@ -546,153 +556,276 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json bool AudioSession::IsNemotronSpeechModel() const { - const auto& cfg = Model().GetConfig(); + const auto& cfg = Model().GetGenAIConfig(); return cfg.model.has_value() && cfg.model->type == "nemotron_speech"; } +void AudioSession::TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const { + if (language.empty()) { + return; + } -void AudioSession::TryNemotronLanguageId(OgaGenerator& generator, - const std::string& language) const { - if (language.empty()) { - return; - } + const auto key = ToLowerAscii(language); + auto it = NemotronLanguageIdMap().find(key); + if (it == NemotronLanguageIdMap().end()) { + return; + } - const auto key = ToLowerAscii(language); - auto it = NemotronLanguageIdMap().find(key); - if (it == NemotronLanguageIdMap().end()) { - return; - } - - - try { - generator.SetRuntimeOption("lang_id", it->second.c_str()); - } catch (const std::exception& e) { - logger_.Log(LogLevel::Error, fmt::format("Failed to set Nemotron language ID: {}", e.what())); - } + try { + generator.SetRuntimeOption("lang_id", it->second.c_str()); + } catch (const std::exception& e) { + logger_.Log(LogLevel::Warning, + fmt::format("Failed to set Nemotron lang_id '{}' for '{}': {}", + it->second, language, e.what())); } +} +void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, + const Request& original_request, + Response& response) { + if (original_request.canceled) { + response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE; + return; + } - void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, - const Requests& original_request, - const Response& response) { - - constexpr size_t kInitialTokenCapacity = 256; + std::optional temperature = req.temperature.has_value() ? req.temperature : session_options_.temperature; - if (original_request.canceled) { - response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE; - return; + std::string language; + if (req.language.has_value()) { + language = *req.language; + } else { + auto session_lang_it = SessionOptions().find("language"); + if (session_lang_it != SessionOptions().end()) { + language = session_lang_it->second; } + } - const float temperature = req.temperature.value_or( - session_options_.temperature.value_or(0.0f) - ); - - const std::string language = req.language.value_or(""); - - std::vector samples = LoadPcmWavAsFloatSamples(req.audio_file_path); + auto samples = LoadPcmWavAsFloatSamples(req.filename); + auto& oga_model = Model().GetOgaModel(); + auto processor = OgaStreamingProcessor::Create(oga_model); + auto tokenizer = OgaTokenizer::Create(oga_model); + auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer); + auto generator_params = OgaGeneratorParams::Create(oga_model); + if (temperature.has_value()) { + generator_params->SetSearchOption("temperature", *temperature); + } else { + generator_params->SetSearchOption("temperature", 0.0f); + } + auto generator = OgaGenerator::Create(oga_model, *generator_params); + TrySetNemotronLanguageId(*generator, language); - auto& oga_model = Model().GetOgaModel(); - - auto processor = OgaStreamingProcessor::Create(oga_model); + auto streaming_callback = CreateCallbackHandler(original_request); + const bool is_streaming = (streaming_callback != nullptr); + std::string response_id = ResponseConverter::GenerateId("audio"); - auto tokenizer_stream = OgaTokenizerStream::Create(Model().GetOgaModel()); + std::string text; + text.reserve(512); + int completion_tokens = 0; - std::vector token_texts; + auto decode_all_tokens = [&]() { + while (!generator->IsDone() && !original_request.canceled) { + generator->GenerateNextToken(); + auto next_tokens = generator->GetNextTokens(); + if (next_tokens.empty()) { + continue; + } - token_texts.reserve(kInitialTokenCapacity); + ++completion_tokens; + const char* decoded = tokenizer_stream->Decode(next_tokens[0]); + if (!decoded || decoded[0] == '\0') { + continue; + } - std::vector> segments; + std::string token(decoded); + if (IsLanguageToken(token)) { + continue; + } - segments.reserve(kInitialSegmentCapacity); + text += token; - int completion_tokens = 0; + if (is_streaming) { + AudioTranscriptionResponse chunk; + chunk.id = response_id; + chunk.text = token; + streaming_callback->PushItem(std::make_unique(nlohmann::json(chunk).dump(), + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + } + } + }; - auto callback = CreateCallbackHandler(original_request); + auto run_one_pass = [&](std::unique_ptr tensors) { + if (!tensors || original_request.canceled) { + return; + } + generator->SetInputs(*tensors); + decode_all_tokens(); + }; - // Process initial tensors with fresh generator + constexpr size_t kNemotronSamplesPerChunk = 1600; // 100ms at 16kHz + for (size_t offset = 0; offset < samples.size() && !original_request.canceled; + offset += kNemotronSamplesPerChunk) { + size_t count = std::min(kNemotronSamplesPerChunk, samples.size() - offset); + run_one_pass(processor->Process(samples.data() + offset, count)); + } + run_one_pass(processor->Flush()); - if (!original_request.canceled) { - // Process initial tensors here + response.finish_reason = original_request.canceled ? FOUNDRY_LOCAL_FINISH_NONE : FOUNDRY_LOCAL_FINISH_STOP; + response.usage.prompt_tokens = 0; + response.usage.completion_tokens = completion_tokens; + response.usage.total_tokens = completion_tokens; - auto tensors = processor->Process(samples.data(), samples.size()); + AudioTranscriptionResponse output; + output.id = response_id; + output.text = std::move(text); + response.items.push_back(std::make_unique(nlohmann::json(output).dump(), + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); +} - if (tensors) { - auto gen_params = OgaGeneratorParams::Create(oga_model); +std::vector AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) { + std::ifstream in(audio_file_path, std::ios::binary); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Failed to open audio file: '{}'", audio_file_path)); + } - gen_params->SetSearchOption("temperature", temperature); + in.seekg(0, std::ios::end); + std::streamoff file_size = in.tellg(); + in.seekg(0, std::ios::beg); + if (file_size < 12) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: too small for RIFF/WAVE header."); + } - auto generator = OgaGenerator::Create(oga_model, *gen_params); + char riff[4]; + uint32_t riff_size = 0; + char wave[4]; + in.read(riff, sizeof(riff)); + in.read(reinterpret_cast(&riff_size), sizeof(riff_size)); + in.read(wave, sizeof(wave)); - TryNemotronLanguageId(*generator, language); + if (!in || std::strncmp(riff, "RIFF", 4) != 0 || std::strncmp(wave, "WAVE", 4) != 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: missing RIFF/WAVE header."); + } - generator->SetInputs(*tensors); + int16_t audio_format = 0; + int16_t channels = 0; + int32_t sample_rate = 0; + int16_t bits_per_sample = 0; + std::vector data; - DecodeTokens(*generator, *tokenizer_stream, token_texts, segments, - callback, original_request, completion_tokens); - } + while (in && in.tellg() < file_size) { + char chunk_id_chars[4]; + uint32_t chunk_size = 0; + in.read(chunk_id_chars, sizeof(chunk_id_chars)); + in.read(reinterpret_cast(&chunk_size), sizeof(chunk_size)); + if (!in) { + break; } - // Flush tensors with fresh generator (Important: avoid done-state reuse bug) - - if (!original_request.canceled) { - auto flush_tensors = processor->Flush(); - - if (flush_tensors) { - - auto gen_params = OgaGeneratorParams::Create(oga_model); - - gen_params->SetSearchOption("temperature", temperature); + const std::string chunk_id(chunk_id_chars, sizeof(chunk_id_chars)); + const std::streamoff chunk_start = in.tellg(); + if (chunk_start < 0 || + chunk_start + static_cast(chunk_size) > + file_size - (chunk_size % 2 == 1 ? 1 : 0)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk."); + } - auto generator = OgaGenerator::Create(oga_model, *gen_params); + if (chunk_id == "fmt ") { + if (chunk_size < 16) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Invalid WAV fmt chunk size {}; expected at least 16.", chunk_size)); + } - TryNemotronLanguageId(*generator, language); + in.read(reinterpret_cast(&audio_format), sizeof(audio_format)); + in.read(reinterpret_cast(&channels), sizeof(channels)); + in.read(reinterpret_cast(&sample_rate), sizeof(sample_rate)); - generator->SetInputs(*flush_tensors); + int32_t byte_rate = 0; + int16_t block_align = 0; + in.read(reinterpret_cast(&byte_rate), sizeof(byte_rate)); + in.read(reinterpret_cast(&block_align), sizeof(block_align)); + in.read(reinterpret_cast(&bits_per_sample), sizeof(bits_per_sample)); - DecodeTokens(*generator, *tokenizer_stream, token_texts, segments, - callback, original_request, completion_tokens); + int32_t remaining = static_cast(chunk_size) - 16; + if (remaining > 0) { + in.seekg(remaining, std::ios::cur); + } + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV fmt chunk."); + } + } else if (chunk_id == "data") { + data.resize(chunk_size); + if (chunk_size > 0) { + in.read(reinterpret_cast(data.data()), chunk_size); + } + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data chunk."); + } + } else { + in.seekg(chunk_size, std::ios::cur); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk."); } } - std::string full_text = JoinTokens(token_texts); - response.items.push_back(BuildSpeechResult(std::move(full_text), std::move(segments))); - - response.finish_reason = original_request.canceled ? FOUNDRY_FINISH_REASON_CANCELED : FOUNDRY_FINISH_REASON_COMPLETED; - - response.usage.prompt_tokens = 0; - - response.usage.completion_tokens = completion_tokens; + if (chunk_size % 2 == 1) { + in.seekg(1, std::ios::cur); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV padding byte."); + } + } + } - response.usage.total_tokens = response.usage.prompt_tokens + response.usage.completion_tokens; + if (channels <= 0 || sample_rate <= 0 || bits_per_sample <= 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Missing or invalid WAV fmt chunk."); + } + if (sample_rate != 16000) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Expected 16kHz WAV input, got {}Hz.", sample_rate)); + } + if (data.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "WAV data chunk is missing or empty."); } - std::vector AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) { + const size_t bytes_per_sample = static_cast(bits_per_sample / 8); + if ((audio_format != 1 || bits_per_sample != 16) && (audio_format != 3 || bits_per_sample != 32)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Unsupported WAV format: audioFormat={}, bitsPerSample={}.", audio_format, bits_per_sample)); + } + if (bytes_per_sample == 0 || data.size() % (bytes_per_sample * static_cast(channels)) != 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data size."); + } - std::ifstream in(audio_file_path, std::ios::binary); + const size_t frame_count = data.size() / (bytes_per_sample * static_cast(channels)); + std::vector samples; + samples.reserve(frame_count); - if (!in) { - FL_THROW("Failed to open audio file: " + audio_file_path); + if (audio_format == 1 && bits_per_sample == 16) { + for (size_t frame = 0; frame < frame_count; ++frame) { + float mixed = 0.0f; + for (int16_t ch = 0; ch < channels; ++ch) { + size_t idx = frame * static_cast(channels) + static_cast(ch); + size_t byte_offset = idx * bytes_per_sample; + int16_t sample = 0; + std::memcpy(&sample, data.data() + byte_offset, sizeof(sample)); + mixed += static_cast(sample) / 32768.0f; + } + samples.push_back(mixed / static_cast(channels)); } + return samples; + } - // TODO: - // 1) Read RIFF/WAVE header, fail if invalid. - // 2) Iterate chunks safely: - // - read chunk id + size - // - bounds-check size - // - if "fmt " and size < 16 => throw (GitOps feedback fix) - // - parse format/channels/sampleRate/bitsPerSample - // - capture "data" bytes - // 3) Enforce sampleRate == 16000 - // 4) Convert: - // - PCM16 -> float [-1,1], average channels to mono if needed - // - (optional parity) float32 PCM -> mono float - // 5) return std::vector - - FL_THROW(FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED, - "LoadPcmWavAsFloatSamples not implemented yet"); -} - + for (size_t frame = 0; frame < frame_count; ++frame) { + float mixed = 0.0f; + for (int16_t ch = 0; ch < channels; ++ch) { + size_t idx = frame * static_cast(channels) + static_cast(ch); + float value = 0.0f; + std::memcpy(&value, data.data() + (idx * bytes_per_sample), sizeof(float)); + mixed += value; + } + samples.push_back(mixed / static_cast(channels)); } + return samples; +} } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h index 1dd913f81..9ac54fca2 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h @@ -19,6 +19,7 @@ struct OgaTokenizerStream; namespace fl { class GenAIModelInstance; +struct AudioTranscriptionRequest; struct AudioItem; struct ItemQueue; struct SpeechSegmentItem; @@ -61,7 +62,7 @@ class AudioSession : public Session { void TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const; - static std::vector LoadPcmWavAsFloatSample(const std::string&& audio_file_path); + static std::vector LoadPcmWavAsFloatSamples(const std::string& audio_file_path); /// Process a streaming audio request: an AudioItem (format descriptor) + an ItemQueue (PCM chunks). void ProcessStreamingAudio(const AudioItem& format_item, ItemQueue& queue,