diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc index 06a597ea9..500c982e6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc @@ -19,9 +19,15 @@ #include "utils.h" #include +#include +#include +#include +#include +#include #include #include #include +#include namespace fl { @@ -71,8 +77,47 @@ std::string JoinTokens(const std::vector& token_texts) { return out; } +const std::unordered_map& NemotronLanguageIdMap() { + static const std::unordered_map kMap = { + {"en", "0"}, {"en-us", "0"}, {"en-gb", "1"}, {"es-es", "2"}, {"es", "3"}, + {"es-us", "3"}, {"zh-cn", "4"}, {"hi", "6"}, {"hi-in", "6"}, {"ar", "7"}, + {"ar-ar", "7"}, {"fr", "8"}, {"fr-fr", "8"}, {"fr-ca", "100"}, {"de", "9"}, + {"de-de", "9"}, {"ja", "10"}, {"ja-jp", "10"}, {"ru", "11"}, {"ru-ru", "11"}, + {"pt-br", "12"}, {"pt", "13"}, {"pt-pt", "13"}, {"ko", "14"}, {"ko-kr", "14"}, + {"it", "15"}, {"it-it", "15"}, {"nl", "16"}, {"nl-nl", "16"}, {"pl", "17"}, + {"pl-pl", "17"}, {"tr", "18"}, {"tr-tr", "18"}, {"uk", "19"}, {"uk-ua", "19"}, + {"ro", "20"}, {"ro-ro", "20"}, {"el", "21"}, {"el-gr", "21"}, {"cs", "22"}, + {"cs-cz", "22"}, {"hu", "23"}, {"hu-hu", "23"}, {"sv", "24"}, {"sv-se", "24"}, + {"da", "25"}, {"da-dk", "25"}, {"fi", "26"}, {"fi-fi", "26"}, {"sk", "28"}, + {"sk-sk", "28"}, {"hr", "29"}, {"hr-hr", "29"}, {"bg", "30"}, {"bg-bg", "30"}, + {"lt", "31"}, {"lt-lt", "31"}, {"th", "32"}, {"th-th", "32"}, {"vi", "33"}, + {"vi-vn", "33"}, {"et", "60"}, {"et-ee", "60"}, {"lv", "61"}, {"lv-lv", "61"}, + {"sl", "62"}, {"sl-si", "62"}, {"he", "64"}, {"he-il", "64"}, {"auto", "101"}, + {"mt", "102"}, {"mt-mt", "102"}, {"nb", "103"}, {"nb-no", "103"}, {"nn", "104"}, + {"nn-no", "104"}, + }; + return kMap; +} + +std::string ToLowerAscii(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return s; +} + +bool IsLanguageToken(const std::string& token) { + size_t start = token.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) { + return false; + } + + size_t end = token.find_last_not_of(" \t\r\n"); + return end >= start && token[start] == '<' && token[end] == '>'; +} + } // namespace + AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { @@ -422,6 +467,13 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, fmt::format("Audio file not found: '{}'", req.filename)); } + // If the model is a Nemotron speech model, process the transcription using the Nemotron-specific method. + // e.g. RNNT models require Nemotron-specific transcription handling. + if (IsNemotronSpeechModel()) { + ProcessNemotronFileTranscription(req, original_request, response); + return; + } + // Build generation options from session defaults SearchOptions options = session_options_; @@ -502,4 +554,278 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json total_tokens, prompt_tokens, completion_tokens)); } + +bool AudioSession::IsNemotronSpeechModel() const { + const auto& cfg = Model().GetGenAIConfig(); + return cfg.model.has_value() && cfg.model->type == "nemotron_speech"; +} + +void AudioSession::TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const { + if (language.empty()) { + return; + } + + const auto key = ToLowerAscii(language); + auto it = NemotronLanguageIdMap().find(key); + if (it == NemotronLanguageIdMap().end()) { + return; + } + + try { + generator.SetRuntimeOption("lang_id", it->second.c_str()); + } catch (const std::exception& e) { + logger_.Log(LogLevel::Warning, + fmt::format("Failed to set Nemotron lang_id '{}' for '{}': {}", + it->second, language, e.what())); + } +} + +void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, + const Request& original_request, + Response& response) { + if (original_request.canceled) { + response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE; + return; + } + + std::optional temperature = req.temperature.has_value() ? req.temperature : session_options_.temperature; + + std::string language; + if (req.language.has_value()) { + language = *req.language; + } else { + auto session_lang_it = SessionOptions().find("language"); + if (session_lang_it != SessionOptions().end()) { + language = session_lang_it->second; + } + } + + auto samples = LoadPcmWavAsFloatSamples(req.filename); + auto& oga_model = Model().GetOgaModel(); + auto processor = OgaStreamingProcessor::Create(oga_model); + auto tokenizer = OgaTokenizer::Create(oga_model); + auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer); + auto generator_params = OgaGeneratorParams::Create(oga_model); + if (temperature.has_value()) { + generator_params->SetSearchOption("temperature", *temperature); + } else { + generator_params->SetSearchOption("temperature", 0.0f); + } + auto generator = OgaGenerator::Create(oga_model, *generator_params); + TrySetNemotronLanguageId(*generator, language); + + auto streaming_callback = CreateCallbackHandler(original_request); + const bool is_streaming = (streaming_callback != nullptr); + std::string response_id = ResponseConverter::GenerateId("audio"); + + std::string text; + text.reserve(512); + int completion_tokens = 0; + + auto decode_all_tokens = [&]() { + while (!generator->IsDone() && !original_request.canceled) { + generator->GenerateNextToken(); + auto next_tokens = generator->GetNextTokens(); + if (next_tokens.empty()) { + continue; + } + + ++completion_tokens; + const char* decoded = tokenizer_stream->Decode(next_tokens[0]); + if (!decoded || decoded[0] == '\0') { + continue; + } + + std::string token(decoded); + if (IsLanguageToken(token)) { + continue; + } + + text += token; + + if (is_streaming) { + AudioTranscriptionResponse chunk; + chunk.id = response_id; + chunk.text = token; + streaming_callback->PushItem(std::make_unique(nlohmann::json(chunk).dump(), + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + } + } + }; + + auto run_one_pass = [&](std::unique_ptr tensors) { + if (!tensors || original_request.canceled) { + return; + } + generator->SetInputs(*tensors); + decode_all_tokens(); + }; + + constexpr size_t kNemotronSamplesPerChunk = 1600; // 100ms at 16kHz + for (size_t offset = 0; offset < samples.size() && !original_request.canceled; + offset += kNemotronSamplesPerChunk) { + size_t count = std::min(kNemotronSamplesPerChunk, samples.size() - offset); + run_one_pass(processor->Process(samples.data() + offset, count)); + } + run_one_pass(processor->Flush()); + + response.finish_reason = original_request.canceled ? FOUNDRY_LOCAL_FINISH_NONE : FOUNDRY_LOCAL_FINISH_STOP; + response.usage.prompt_tokens = 0; + response.usage.completion_tokens = completion_tokens; + response.usage.total_tokens = completion_tokens; + + AudioTranscriptionResponse output; + output.id = response_id; + output.text = std::move(text); + response.items.push_back(std::make_unique(nlohmann::json(output).dump(), + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); +} + +std::vector AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) { + std::ifstream in(audio_file_path, std::ios::binary); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Failed to open audio file: '{}'", audio_file_path)); + } + + in.seekg(0, std::ios::end); + std::streamoff file_size = in.tellg(); + in.seekg(0, std::ios::beg); + if (file_size < 12) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: too small for RIFF/WAVE header."); + } + + char riff[4]; + uint32_t riff_size = 0; + char wave[4]; + in.read(riff, sizeof(riff)); + in.read(reinterpret_cast(&riff_size), sizeof(riff_size)); + in.read(wave, sizeof(wave)); + + if (!in || std::strncmp(riff, "RIFF", 4) != 0 || std::strncmp(wave, "WAVE", 4) != 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: missing RIFF/WAVE header."); + } + + int16_t audio_format = 0; + int16_t channels = 0; + int32_t sample_rate = 0; + int16_t bits_per_sample = 0; + std::vector data; + + while (in && in.tellg() < file_size) { + char chunk_id_chars[4]; + uint32_t chunk_size = 0; + in.read(chunk_id_chars, sizeof(chunk_id_chars)); + in.read(reinterpret_cast(&chunk_size), sizeof(chunk_size)); + if (!in) { + break; + } + + const std::string chunk_id(chunk_id_chars, sizeof(chunk_id_chars)); + const std::streamoff chunk_start = in.tellg(); + if (chunk_start < 0 || + chunk_start + static_cast(chunk_size) > + file_size - (chunk_size % 2 == 1 ? 1 : 0)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk."); + } + + if (chunk_id == "fmt ") { + if (chunk_size < 16) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Invalid WAV fmt chunk size {}; expected at least 16.", chunk_size)); + } + + in.read(reinterpret_cast(&audio_format), sizeof(audio_format)); + in.read(reinterpret_cast(&channels), sizeof(channels)); + in.read(reinterpret_cast(&sample_rate), sizeof(sample_rate)); + + int32_t byte_rate = 0; + int16_t block_align = 0; + in.read(reinterpret_cast(&byte_rate), sizeof(byte_rate)); + in.read(reinterpret_cast(&block_align), sizeof(block_align)); + in.read(reinterpret_cast(&bits_per_sample), sizeof(bits_per_sample)); + + int32_t remaining = static_cast(chunk_size) - 16; + if (remaining > 0) { + in.seekg(remaining, std::ios::cur); + } + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV fmt chunk."); + } + } else if (chunk_id == "data") { + data.resize(chunk_size); + if (chunk_size > 0) { + in.read(reinterpret_cast(data.data()), chunk_size); + } + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data chunk."); + } + } else { + in.seekg(chunk_size, std::ios::cur); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk."); + } + } + + if (chunk_size % 2 == 1) { + in.seekg(1, std::ios::cur); + if (!in) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV padding byte."); + } + } + } + + if (channels <= 0 || sample_rate <= 0 || bits_per_sample <= 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Missing or invalid WAV fmt chunk."); + } + if (sample_rate != 16000) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Expected 16kHz WAV input, got {}Hz.", sample_rate)); + } + if (data.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "WAV data chunk is missing or empty."); + } + + const size_t bytes_per_sample = static_cast(bits_per_sample / 8); + if ((audio_format != 1 || bits_per_sample != 16) && (audio_format != 3 || bits_per_sample != 32)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + fmt::format("Unsupported WAV format: audioFormat={}, bitsPerSample={}.", audio_format, bits_per_sample)); + } + if (bytes_per_sample == 0 || data.size() % (bytes_per_sample * static_cast(channels)) != 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data size."); + } + + const size_t frame_count = data.size() / (bytes_per_sample * static_cast(channels)); + std::vector samples; + samples.reserve(frame_count); + + if (audio_format == 1 && bits_per_sample == 16) { + for (size_t frame = 0; frame < frame_count; ++frame) { + float mixed = 0.0f; + for (int16_t ch = 0; ch < channels; ++ch) { + size_t idx = frame * static_cast(channels) + static_cast(ch); + size_t byte_offset = idx * bytes_per_sample; + int16_t sample = 0; + std::memcpy(&sample, data.data() + byte_offset, sizeof(sample)); + mixed += static_cast(sample) / 32768.0f; + } + samples.push_back(mixed / static_cast(channels)); + } + return samples; + } + + for (size_t frame = 0; frame < frame_count; ++frame) { + float mixed = 0.0f; + for (int16_t ch = 0; ch < channels; ++ch) { + size_t idx = frame * static_cast(channels) + static_cast(ch); + float value = 0.0f; + std::memcpy(&value, data.data() + (idx * bytes_per_sample), sizeof(float)); + mixed += value; + } + samples.push_back(mixed / static_cast(channels)); + } + + return samples; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h index 1a35d096a..9ac54fca2 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h @@ -19,6 +19,7 @@ struct OgaTokenizerStream; namespace fl { class GenAIModelInstance; +struct AudioTranscriptionRequest; struct AudioItem; struct ItemQueue; struct SpeechSegmentItem; @@ -53,6 +54,16 @@ class AudioSession : public Session { void ProcessAudioTranscriptionJson(const std::string& request_json, const Request& original_request, Response& response); + bool IsNemotronSpeechModel() const; + + void ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req, + const Request& original_request, + Response& response); + + void TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const; + + static std::vector LoadPcmWavAsFloatSamples(const std::string& audio_file_path); + /// Process a streaming audio request: an AudioItem (format descriptor) + an ItemQueue (PCM chunks). void ProcessStreamingAudio(const AudioItem& format_item, ItemQueue& queue, const Request& request, Response& response);