Return token stop timestamp for CTC decoding.

sherpa-onnx/csrc/offline-ctc-decoder.h

@@ -15,17 +15,26 @@ struct OfflineCtcDecoderResult {
   /// The decoded token IDs
   std::vector<int64_t> tokens;
 
+  /// timestamps[i] contains the output frame index where tokens[i] is decoded.
+  /// Note: The index is after subsampling
+  ///
+  /// tokens.size() == timestamps.size()
+  std::vector<int32_t> timestamps;
+
+  /// stop_timestamps[i] - timestamps[i] is the duration of the i-th token
+  /// in terms of number of output frames
+  ///
+  /// tokens.size() == stop_timestamps.size()
+  std::vector<int32_t> stop_timestamps;
+
   /// The decoded word IDs
   /// Note: tokens.size() is usually not equal to words.size()
   /// words is empty for greedy search decoding.
   /// it is not empty when an HLG graph or an HLG graph is used.
   std::vector<int32_t> words;
 
-  /// timestamps[i] contains the output frame index where tokens[i] is decoded.
-  /// Note: The index is after subsampling
-  ///
-  /// tokens.size() == timestamps.size()
-  std::vector<int32_t> timestamps;
+  /// word_start_timestamps.size() == words.size()
+  std::vector<int32_t> word_start_timestamps;
 };
 
 class OfflineCtcDecoder {

sherpa-onnx/csrc/offline-ctc-fst-decoder.cc

@@ -88,13 +88,19 @@ static OfflineCtcDecoderResult DecodeOne(kaldi_decoder::FasterDecoder *decoder,
   auto cur_state = decoded.Start();
 
   int32_t blank_id = 0;
+  int32_t prev = -1;
+  int32_t t = 0;
 
-  for (int32_t t = 0, prev = -1; decoded.NumArcs(cur_state) == 1; ++t) {
+  for (; decoded.NumArcs(cur_state) == 1; ++t) {
     fst::ArcIterator<fst::Fst<fst::LatticeArc>> iter(decoded, cur_state);
     const auto &arc = iter.Value();
 
     cur_state = arc.nextstate;
 
+    if (prev != -1 && prev != 0 && prev != blank_id + 1 && arc.ilabel != prev) {
+      r.stop_timestamps.push_back(t);
+    }
+
     if (arc.ilabel == prev) {
       continue;
     }
@@ -110,12 +116,21 @@ static OfflineCtcDecoderResult DecodeOne(kaldi_decoder::FasterDecoder *decoder,
     r.tokens.push_back(arc.ilabel - 1);
     if (arc.olabel != 0) {
       r.words.push_back(arc.olabel);
+      r.word_start_timestamps.push_back(t);
     }
 
     r.timestamps.push_back(t);
     prev = arc.ilabel;
   }
 
+  if (r.timestamps.size() != r.stop_timestamps.size()) {
+    r.stop_timestamps.push_back(t);
+  }
+
+  if (r.timestamps.size() != r.stop_timestamps.size()) {
+    SHERPA_ONNX_LOGE("something bad happened");
+  }
+
   return r;
 }
 

sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.cc

@@ -29,23 +29,36 @@ std::vector<OfflineCtcDecoderResult> OfflineCtcGreedySearchDecoder::Decode(
         log_probs.GetTensorData<float>() + b * num_frames * vocab_size;
 
     OfflineCtcDecoderResult r;
-    int64_t prev_id = -1;
+    int32_t prev_id = -1;
+    int32_t t = 0;
 
-    for (int32_t t = 0; t != static_cast<int32_t>(p_log_probs_length[b]); ++t) {
+    for (; t != static_cast<int32_t>(p_log_probs_length[b]); ++t) {
       auto y = static_cast<int64_t>(std::distance(
           static_cast<const float *>(p_log_probs),
           std::max_element(
               static_cast<const float *>(p_log_probs),
               static_cast<const float *>(p_log_probs) + vocab_size)));
       p_log_probs += vocab_size;
 
+      if (prev_id != -1 && prev_id != blank_id_ && y != prev_id) {
+        r.stop_timestamps.push_back(t);
+      }
+
       if (y != blank_id_ && y != prev_id) {
         r.tokens.push_back(y);
         r.timestamps.push_back(t);
       }
       prev_id = y;
     }  // for (int32_t t = 0; ...)
 
+    if (r.timestamps.size() != r.stop_timestamps.size()) {
+      r.stop_timestamps.push_back(t);
+    }
+
+    if (r.timestamps.size() != r.stop_timestamps.size()) {
+      SHERPA_ONNX_LOGE("something bad happened");
+    }
+
     ans.push_back(std::move(r));
   }
   return ans;

sherpa-onnx/csrc/offline-recognizer-ctc-impl.h

@@ -34,6 +34,9 @@ static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src,
   OfflineRecognitionResult r;
   r.tokens.reserve(src.tokens.size());
   r.timestamps.reserve(src.timestamps.size());
+  r.stop_timestamps.reserve(src.stop_timestamps.size());
+
+  r.word_start_timestamps.reserve(src.word_start_timestamps.size());
 
   std::string text;
 
@@ -65,6 +68,16 @@ static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src,
     r.timestamps.push_back(time);
   }
 
+  for (auto t : src.stop_timestamps) {
+    float time = frame_shift_s * t;
+    r.stop_timestamps.push_back(time);
+  }
+
+  for (auto t : src.word_start_timestamps) {
+    float time = frame_shift_s * t;
+    r.word_start_timestamps.push_back(time);
+  }
+
   r.words = std::move(src.words);
 
   return r;

sherpa-onnx/csrc/offline-stream.cc

@@ -14,6 +14,7 @@
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/offline-recognizer.h"
 #include "sherpa-onnx/csrc/resample.h"
+#include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {
 
@@ -305,26 +306,16 @@ std::string OfflineRecognitionResult::AsJsonString() const {
   os << "\"" << text << "\""
      << ", ";
 
-  os << "\""
-     << "timestamps"
-     << "\""
-     << ": ";
-  os << "[";
-
-  std::string sep = "";
-  for (auto t : timestamps) {
-    os << sep << std::fixed << std::setprecision(2) << t;
-    sep = ", ";
-  }
-  os << "], ";
+  os << "\"timestamps\": " << VecToString(timestamps, 2) << ", ";
+  os << "\"stop_timestamps\": " << VecToString(stop_timestamps, 2) << ", ";
 
   os << "\""
      << "tokens"
      << "\""
      << ":";
   os << "[";
 
-  sep = "";
+  std::string sep = "";
   auto oldFlags = os.flags();
   for (const auto &t : tokens) {
     if (t.size() == 1 && static_cast<uint8_t>(t[0]) > 0x7f) {
@@ -341,19 +332,10 @@ std::string OfflineRecognitionResult::AsJsonString() const {
   }
   os << "], ";
 
-  sep = "";
+  os << "\"words\": " << VecToString(words, 0) << ", ";
 
-  os << "\""
-     << "words"
-     << "\""
-     << ": ";
-  os << "[";
-  for (int32_t w : words) {
-    os << sep << w;
-    sep = ", ";
-  }
+  os << "\"word_start_timestamps\": " << VecToString(word_start_timestamps, 2);
 
-  os << "]";
   os << "}";
 
   return os.str();

sherpa-onnx/csrc/offline-stream.h

@@ -28,10 +28,29 @@ struct OfflineRecognitionResult {
 
   /// timestamps.size() == tokens.size()
   /// timestamps[i] records the time in seconds when tokens[i] is decoded.
+  ///
+  /// Note: It is the start time stamp of a token.
+  ///
+  /// It is empty if the model does not support time stamp information.
   std::vector<float> timestamps;
 
+  /// It is not empty for CTC models.
+  /// It is empty for non-CTC models.
+  /// If it is not empty, then stop_timestamps.size() == timestamps.size()
+  std::vector<float> stop_timestamps;
+
+  /// It is not empty for CTC models with a HL or HLG decoding graph
+  /// It is empty for non-CTC models.
+  ///
+  /// If not empty, it contains word IDs. You have to use words.txt
+  /// to map word IDs to word symbols.
   std::vector<int32_t> words;
 
+  /// If not empty, word_start_timestamps[i] is the start time of words[i].
+  ///
+  /// words.size() == word_start_timestamps.size()
+  std::vector<float> word_start_timestamps;
+
   std::string AsJsonString() const;
 };
 

sherpa-onnx/csrc/online-recognizer.cc

@@ -15,41 +15,10 @@
 #include <vector>
 
 #include "sherpa-onnx/csrc/online-recognizer-impl.h"
+#include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {
 
-/// Helper for `OnlineRecognizerResult::AsJsonString()`
-template <typename T>
-std::string VecToString(const std::vector<T> &vec, int32_t precision = 6) {
-  std::ostringstream oss;
-  if (precision != 0) {
-    oss << std::fixed << std::setprecision(precision);
-  }
-  oss << "[";
-  std::string sep = "";
-  for (const auto &item : vec) {
-    oss << sep << item;
-    sep = ", ";
-  }
-  oss << "]";
-  return oss.str();
-}
-
-/// Helper for `OnlineRecognizerResult::AsJsonString()`
-template <>  // explicit specialization for T = std::string
-std::string VecToString<std::string>(const std::vector<std::string> &vec,
-                                     int32_t) {  // ignore 2nd arg
-  std::ostringstream oss;
-  oss << "[";
-  std::string sep = "";
-  for (const auto &item : vec) {
-    oss << sep << "\"" << item << "\"";
-    sep = ", ";
-  }
-  oss << "]";
-  return oss.str();
-}
-
 std::string OnlineRecognizerResult::AsJsonString() const {
   std::ostringstream os;
   os << "{ ";

sherpa-onnx/csrc/text-utils.cc

@@ -10,6 +10,7 @@
 #include <algorithm>
 #include <cctype>
 #include <cstdint>
+#include <iomanip>
 #include <limits>
 #include <sstream>
 #include <string>
@@ -397,4 +398,38 @@ void ToLowerCase(std::string *in_out) {
                  [](unsigned char c) { return std::tolower(c); });
 }
 
+template <typename T>
+std::string VecToString(const std::vector<T> &vec, int32_t precision /*= 6*/) {
+  std::ostringstream os;
+  if (precision != 0) {
+    os << std::fixed << std::setprecision(precision);
+  }
+  os << "[";
+  std::string sep = "";
+  for (const auto &item : vec) {
+    os << sep << item;
+    sep = ", ";
+  }
+  os << "]";
+  return os.str();
+}
+
+template std::string VecToString<int32_t>(const std::vector<int32_t> &vec,
+                                          int32_t precision /*= 6*/);
+
+template std::string VecToString<float>(const std::vector<float> &vec,
+                                        int32_t precision /*= 6*/);
+
+std::string VecToString(const std::vector<std::string> &vec) {
+  std::ostringstream os;
+  os << "[";
+  std::string sep = "";
+  for (const auto &item : vec) {
+    os << sep << "\"" << item << "\"";
+    sep = ", ";
+  }
+  os << "]";
+  return os.str();
+}
+
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/text-utils.h

@@ -124,6 +124,11 @@ std::vector<std::string> SplitUtf8(const std::string &text);
 std::string ToLowerCase(const std::string &s);
 void ToLowerCase(std::string *in_out);
 
+template <typename T>
+std::string VecToString(const std::vector<T> &vec, int32_t precision = 6);
+
+std::string VecToString(const std::vector<std::string> &vec);
+
 }  // namespace sherpa_onnx
 
 #endif  // SHERPA_ONNX_CSRC_TEXT_UTILS_H_

sherpa-onnx/python/csrc/offline-stream.cc

@@ -37,7 +37,13 @@ static void PybindOfflineRecognitionResult(py::module *m) {  // NOLINT
       .def_property_readonly("words",
                              [](const PyClass &self) { return self.words; })
       .def_property_readonly(
-          "timestamps", [](const PyClass &self) { return self.timestamps; });
+          "word_start_timestamps",
+          [](const PyClass &self) { return self.word_start_timestamps; })
+      .def_property_readonly(
+          "timestamps", [](const PyClass &self) { return self.timestamps; })
+      .def_property_readonly("stop_timestamps", [](const PyClass &self) {
+        return self.stop_timestamps;
+      });
 }
 
 void PybindOfflineStream(py::module *m) {