Skip to content

support file-based ASR for nemotron streaming model #885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rui-ren wants to merge 2 commits into
base: main
Choose a base branch
from
+337 −0
Open

support file-based ASR for nemotron streaming model #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77170bd
add file based support for streaming model
Jul 14, 2026
d5eafd6
update the cpp code for streaming model
rui-ren Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
326 changes: 326 additions & 0 deletions sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,15 @@
#include "utils.h"

#include <filesystem>
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <fmt/format.h>
#include <nlohmann/json.hpp>
#include <ort_genai.h>
#include <unordered_map>

namespace fl {

Expand Down Expand Up @@ -71,8 +77,47 @@ std::string JoinTokens(const std::vector<std::string>& token_texts) {
return out;
}

const std::unordered_map<std::string, std::string>& NemotronLanguageIdMap() {
static const std::unordered_map<std::string, std::string> kMap = {
{"en", "0"}, {"en-us", "0"}, {"en-gb", "1"}, {"es-es", "2"}, {"es", "3"},
{"es-us", "3"}, {"zh-cn", "4"}, {"hi", "6"}, {"hi-in", "6"}, {"ar", "7"},
{"ar-ar", "7"}, {"fr", "8"}, {"fr-fr", "8"}, {"fr-ca", "100"}, {"de", "9"},
{"de-de", "9"}, {"ja", "10"}, {"ja-jp", "10"}, {"ru", "11"}, {"ru-ru", "11"},
{"pt-br", "12"}, {"pt", "13"}, {"pt-pt", "13"}, {"ko", "14"}, {"ko-kr", "14"},
{"it", "15"}, {"it-it", "15"}, {"nl", "16"}, {"nl-nl", "16"}, {"pl", "17"},
{"pl-pl", "17"}, {"tr", "18"}, {"tr-tr", "18"}, {"uk", "19"}, {"uk-ua", "19"},
{"ro", "20"}, {"ro-ro", "20"}, {"el", "21"}, {"el-gr", "21"}, {"cs", "22"},
{"cs-cz", "22"}, {"hu", "23"}, {"hu-hu", "23"}, {"sv", "24"}, {"sv-se", "24"},
{"da", "25"}, {"da-dk", "25"}, {"fi", "26"}, {"fi-fi", "26"}, {"sk", "28"},
{"sk-sk", "28"}, {"hr", "29"}, {"hr-hr", "29"}, {"bg", "30"}, {"bg-bg", "30"},
{"lt", "31"}, {"lt-lt", "31"}, {"th", "32"}, {"th-th", "32"}, {"vi", "33"},
{"vi-vn", "33"}, {"et", "60"}, {"et-ee", "60"}, {"lv", "61"}, {"lv-lv", "61"},
{"sl", "62"}, {"sl-si", "62"}, {"he", "64"}, {"he-il", "64"}, {"auto", "101"},
{"mt", "102"}, {"mt-mt", "102"}, {"nb", "103"}, {"nb-no", "103"}, {"nn", "104"},
{"nn-no", "104"},
};
return kMap;
}

std::string ToLowerAscii(std::string s) {
std::transform(s.begin(), s.end(), s.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return s;
}

bool IsLanguageToken(const std::string& token) {
size_t start = token.find_first_not_of(" \t\r\n");
if (start == std::string::npos) {
return false;
}

size_t end = token.find_last_not_of(" \t\r\n");
return end >= start && token[start] == '<' && token[end] == '>';
}

} // namespace


AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model,
ILogger& logger, ITelemetry& telemetry)
: Session(catalog_model, logger, telemetry), logger_(logger), model_(model) {
Expand Down Expand Up @@ -422,6 +467,13 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json
FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, fmt::format("Audio file not found: '{}'", req.filename));
}

// If the model is a Nemotron speech model, process the transcription using the Nemotron-specific method.
// e.g. RNNT models require Nemotron-specific transcription handling.
if (IsNemotronSpeechModel()) {

@kunal-vaishnavi kunal-vaishnavi Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I presume that the v2 SDK already supported Nemotron when it was merged into main. Why do we now need an early return to support it?

ProcessNemotronFileTranscription(req, original_request, response);
return;
}

// Build generation options from session defaults
SearchOptions options = session_options_;

Expand Down Expand Up @@ -502,4 +554,278 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json
total_tokens, prompt_tokens, completion_tokens));
}


bool AudioSession::IsNemotronSpeechModel() const {
const auto& cfg = Model().GetGenAIConfig();
return cfg.model.has_value() && cfg.model->type == "nemotron_speech";
}

void AudioSession::TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const {
if (language.empty()) {
return;
}

const auto key = ToLowerAscii(language);
auto it = NemotronLanguageIdMap().find(key);
if (it == NemotronLanguageIdMap().end()) {
return;
}

try {
generator.SetRuntimeOption("lang_id", it->second.c_str());
} catch (const std::exception& e) {
logger_.Log(LogLevel::Warning,
fmt::format("Failed to set Nemotron lang_id '{}' for '{}': {}",
it->second, language, e.what()));
}
}

void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req,
const Request& original_request,
Response& response) {
if (original_request.canceled) {
response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE;
return;
}

std::optional<float> temperature = req.temperature.has_value() ? req.temperature : session_options_.temperature;

std::string language;
if (req.language.has_value()) {
language = *req.language;
} else {
auto session_lang_it = SessionOptions().find("language");
if (session_lang_it != SessionOptions().end()) {
language = session_lang_it->second;
}
}

auto samples = LoadPcmWavAsFloatSamples(req.filename);
auto& oga_model = Model().GetOgaModel();
auto processor = OgaStreamingProcessor::Create(oga_model);
auto tokenizer = OgaTokenizer::Create(oga_model);
auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer);
auto generator_params = OgaGeneratorParams::Create(oga_model);
if (temperature.has_value()) {
generator_params->SetSearchOption("temperature", *temperature);

@kunal-vaishnavi kunal-vaishnavi Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we simplify this if-else branching to the following?

generator_params->SetSearchOption("temperature", temperature.value_or(0.0f));

} else {
generator_params->SetSearchOption("temperature", 0.0f);
}
auto generator = OgaGenerator::Create(oga_model, *generator_params);
TrySetNemotronLanguageId(*generator, language);

auto streaming_callback = CreateCallbackHandler(original_request);
const bool is_streaming = (streaming_callback != nullptr);
std::string response_id = ResponseConverter::GenerateId("audio");

std::string text;
text.reserve(512);
int completion_tokens = 0;

auto decode_all_tokens = [&]() {

@kunal-vaishnavi kunal-vaishnavi Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we put this in a named helper function rather than use a lambda function?

while (!generator->IsDone() && !original_request.canceled) {
generator->GenerateNextToken();
auto next_tokens = generator->GetNextTokens();
if (next_tokens.empty()) {
continue;
}

++completion_tokens;
const char* decoded = tokenizer_stream->Decode(next_tokens[0]);
if (!decoded || decoded[0] == '\0') {
continue;
}

std::string token(decoded);
if (IsLanguageToken(token)) {
continue;
}

text += token;

if (is_streaming) {
AudioTranscriptionResponse chunk;
chunk.id = response_id;
chunk.text = token;
streaming_callback->PushItem(std::make_unique<TextItem>(nlohmann::json(chunk).dump(),
FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON));
}
}
};

auto run_one_pass = [&](std::unique_ptr<OgaNamedTensors> tensors) {

@kunal-vaishnavi kunal-vaishnavi Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment here

if (!tensors || original_request.canceled) {
return;
}
generator->SetInputs(*tensors);
decode_all_tokens();
};

constexpr size_t kNemotronSamplesPerChunk = 1600; // 100ms at 16kHz
for (size_t offset = 0; offset < samples.size() && !original_request.canceled;
offset += kNemotronSamplesPerChunk) {
size_t count = std::min(kNemotronSamplesPerChunk, samples.size() - offset);
run_one_pass(processor->Process(samples.data() + offset, count));
}
run_one_pass(processor->Flush());

response.finish_reason = original_request.canceled ? FOUNDRY_LOCAL_FINISH_NONE : FOUNDRY_LOCAL_FINISH_STOP;
response.usage.prompt_tokens = 0;
response.usage.completion_tokens = completion_tokens;
response.usage.total_tokens = completion_tokens;

AudioTranscriptionResponse output;
output.id = response_id;
output.text = std::move(text);
response.items.push_back(std::make_unique<TextItem>(nlohmann::json(output).dump(),
FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON));
}

std::vector<float> AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) {

@kunal-vaishnavi kunal-vaishnavi Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From Copilot:

Good addition overall — the Nemotron transcription path is clean, the WAV parser is thorough, and keeping a single generator across chunks is the right fix for truncated output. A few things worth addressing before merging:

🔴 Blocking issues

  1. WAV chunk bounds check is subtly wrong (LoadPcmWavAsFloatSamples, chunk-walking loop)
    The bounds guard chunk_start + chunk_size > file_size - (chunk_size % 2 == 1 ? 1 : 0) mixes the padding-byte adjustment into the main bounds check, which can undercount available space and let a crafted file slip through. The padding byte lives after the chunk data, so it shouldn't reduce the window used to validate the chunk itself. Suggested fix:

    // Before: chunk_start + chunk_size > file_size - (chunk_size % 2 == 1 ? 1 : 0)
// After:
if (chunk_start < 0 ||
    chunk_start + static_cast<std::streamoff>(chunk_size) > file_size) {
  FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk.");
}

    The odd-byte padding seek that follows handles alignment correctly as-is.

  2. Format validation happens after full data load (LoadPcmWavAsFloatSamples, data chunk branch)
    data.resize(chunk_size) allocates the full audio buffer before sample rate and format have been validated. For a large or malformed file, this could allocate hundreds of MB only to throw immediately afterward. Consider moving the sample_rate, audio_format, and bits_per_sample checks to right after the fmt chunk is parsed, before the data chunk is even encountered.

🟡 Suggestions

  1. Gap at language ID 5 in NemotronLanguageIdMap — IDs jump from 4 (zh-cn) to 6 (hi). If this is intentional (reserved/unsupported language), a brief comment would prevent future contributors from assuming it's a typo.

  2. prompt_tokens hardcoded to 0 in ProcessNemotronFileTranscription — the existing path computes this from the processor. If it's not meaningful for Nemotron, a // TODO explaining why would help maintain parity expectations with the OpenAI usage spec.

  3. decode_all_tokens lambda and thread-safety — the lambda captures generator by reference and is safe as used today (same-thread, same scope). A short comment noting it must not be moved off the creation thread would protect against future refactors.

std::ifstream in(audio_file_path, std::ios::binary);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Failed to open audio file: '{}'", audio_file_path));
}

in.seekg(0, std::ios::end);
std::streamoff file_size = in.tellg();
in.seekg(0, std::ios::beg);
if (file_size < 12) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: too small for RIFF/WAVE header.");
}

char riff[4];
uint32_t riff_size = 0;
char wave[4];
in.read(riff, sizeof(riff));
in.read(reinterpret_cast<char*>(&riff_size), sizeof(riff_size));
in.read(wave, sizeof(wave));

if (!in || std::strncmp(riff, "RIFF", 4) != 0 || std::strncmp(wave, "WAVE", 4) != 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: missing RIFF/WAVE header.");
}

int16_t audio_format = 0;
int16_t channels = 0;
int32_t sample_rate = 0;
int16_t bits_per_sample = 0;
std::vector<uint8_t> data;

while (in && in.tellg() < file_size) {
char chunk_id_chars[4];
uint32_t chunk_size = 0;
in.read(chunk_id_chars, sizeof(chunk_id_chars));
in.read(reinterpret_cast<char*>(&chunk_size), sizeof(chunk_size));
if (!in) {
break;
}

const std::string chunk_id(chunk_id_chars, sizeof(chunk_id_chars));
const std::streamoff chunk_start = in.tellg();
if (chunk_start < 0 ||
chunk_start + static_cast<std::streamoff>(chunk_size) >
file_size - (chunk_size % 2 == 1 ? 1 : 0)) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk.");
}

if (chunk_id == "fmt ") {
if (chunk_size < 16) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Invalid WAV fmt chunk size {}; expected at least 16.", chunk_size));
}

in.read(reinterpret_cast<char*>(&audio_format), sizeof(audio_format));
in.read(reinterpret_cast<char*>(&channels), sizeof(channels));
in.read(reinterpret_cast<char*>(&sample_rate), sizeof(sample_rate));

int32_t byte_rate = 0;
int16_t block_align = 0;
in.read(reinterpret_cast<char*>(&byte_rate), sizeof(byte_rate));
in.read(reinterpret_cast<char*>(&block_align), sizeof(block_align));
in.read(reinterpret_cast<char*>(&bits_per_sample), sizeof(bits_per_sample));

int32_t remaining = static_cast<int32_t>(chunk_size) - 16;
if (remaining > 0) {
in.seekg(remaining, std::ios::cur);
}
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV fmt chunk.");
}
} else if (chunk_id == "data") {
data.resize(chunk_size);
if (chunk_size > 0) {
in.read(reinterpret_cast<char*>(data.data()), chunk_size);
}
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data chunk.");
}
} else {
in.seekg(chunk_size, std::ios::cur);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk.");
}
}

if (chunk_size % 2 == 1) {
in.seekg(1, std::ios::cur);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV padding byte.");
}
}
}

if (channels <= 0 || sample_rate <= 0 || bits_per_sample <= 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Missing or invalid WAV fmt chunk.");
}
if (sample_rate != 16000) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Expected 16kHz WAV input, got {}Hz.", sample_rate));
}
if (data.empty()) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "WAV data chunk is missing or empty.");
}

const size_t bytes_per_sample = static_cast<size_t>(bits_per_sample / 8);
if ((audio_format != 1 || bits_per_sample != 16) && (audio_format != 3 || bits_per_sample != 32)) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Unsupported WAV format: audioFormat={}, bitsPerSample={}.", audio_format, bits_per_sample));
}
if (bytes_per_sample == 0 || data.size() % (bytes_per_sample * static_cast<size_t>(channels)) != 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data size.");
}

const size_t frame_count = data.size() / (bytes_per_sample * static_cast<size_t>(channels));
std::vector<float> samples;
samples.reserve(frame_count);

if (audio_format == 1 && bits_per_sample == 16) {
for (size_t frame = 0; frame < frame_count; ++frame) {
float mixed = 0.0f;
for (int16_t ch = 0; ch < channels; ++ch) {
size_t idx = frame * static_cast<size_t>(channels) + static_cast<size_t>(ch);
size_t byte_offset = idx * bytes_per_sample;
int16_t sample = 0;
std::memcpy(&sample, data.data() + byte_offset, sizeof(sample));
mixed += static_cast<float>(sample) / 32768.0f;
}
samples.push_back(mixed / static_cast<float>(channels));
}
return samples;
}

for (size_t frame = 0; frame < frame_count; ++frame) {
float mixed = 0.0f;
for (int16_t ch = 0; ch < channels; ++ch) {
size_t idx = frame * static_cast<size_t>(channels) + static_cast<size_t>(ch);
float value = 0.0f;
std::memcpy(&value, data.data() + (idx * bytes_per_sample), sizeof(float));
mixed += value;
}
samples.push_back(mixed / static_cast<float>(channels));
}

return samples;
}

} // namespace fl
11 changes: 11 additions & 0 deletions sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ struct OgaTokenizerStream;
namespace fl {

class GenAIModelInstance;
struct AudioTranscriptionRequest;
struct AudioItem;
struct ItemQueue;
struct SpeechSegmentItem;
Expand Down Expand Up @@ -53,6 +54,16 @@ class AudioSession : public Session {
void ProcessAudioTranscriptionJson(const std::string& request_json, const Request& original_request,
Response& response);

bool IsNemotronSpeechModel() const;

void ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req,
const Request& original_request,
Response& response);

void TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const;

static std::vector<float> LoadPcmWavAsFloatSamples(const std::string& audio_file_path);

/// Process a streaming audio request: an AudioItem (format descriptor) + an ItemQueue (PCM chunks).
void ProcessStreamingAudio(const AudioItem& format_item, ItemQueue& queue,
const Request& request, Response& response);
Expand Down