Merge pull request #571 from IENT/feature/ParseLhvCExtradataFromMp4Co…

…ntainers

Add lhvC parsing (MH-HEVC in MP4)

YUViewLib/src/common/Functions.cpp

@@ -168,6 +168,16 @@ std::string toLower(std::string str)
   return str;
 }
 
+ByteVector readData(std::istream &istream, const size_t nrBytes)
+{
+  ByteVector data;
+  data.resize(nrBytes);
+  istream.read(reinterpret_cast<char *>(data.data()), nrBytes);
+  const auto nrBytesActuallyRead = istream.gcount();
+  data.resize(nrBytesActuallyRead);
+  return data;
+}
+
 std::optional<unsigned long> toUnsigned(const std::string &text)
 {
   try

YUViewLib/src/common/Functions.h

@@ -34,6 +34,7 @@
 
 #include <common/Typedef.h>
 
+#include <istream>
 #include <optional>
 
 namespace functions
@@ -64,6 +65,7 @@ QString formatDataSize(double size, bool isBits = false);
 
 QStringList toQStringList(const std::vector<std::string> &stringVec);
 std::string toLower(std::string str);
+ByteVector  readData(std::istream &istream, const size_t nrBytes);
 
 inline std::string boolToString(bool b)
 {

YUViewLib/src/ffmpeg/AVInputFormatWrapper.cpp

@@ -65,6 +65,31 @@ AVInputFormatWrapper::AVInputFormatWrapper(AVInputFormat *f, LibraryVersion v)
   this->update();
 }
 
+QString AVInputFormatWrapper::getName() const
+{
+  return this->name;
+}
+
+QString AVInputFormatWrapper::getLongName() const
+{
+  return this->long_name;
+}
+
+int AVInputFormatWrapper::getFlags() const
+{
+  return this->flags;
+}
+
+QString AVInputFormatWrapper::getExtensions() const
+{
+  return this->extensions;
+}
+
+QString AVInputFormatWrapper::getMimeType() const
+{
+  return this->mime_type;
+}
+
 void AVInputFormatWrapper::update()
 {
   if (this->fmt == nullptr)

YUViewLib/src/ffmpeg/AVInputFormatWrapper.h

@@ -43,6 +43,12 @@ class AVInputFormatWrapper
   AVInputFormatWrapper();
   AVInputFormatWrapper(AVInputFormat *f, LibraryVersion v);
 
+  QString getName() const;
+  QString getLongName() const;
+  int     getFlags() const;
+  QString getExtensions() const;
+  QString getMimeType() const;
+
   explicit operator bool() const { return fmt != nullptr; };
 
 private:

YUViewLib/src/filesource/FileSourceFFmpegFile.cpp

@@ -34,6 +34,7 @@
 
 #include <QProgressDialog>
 #include <QSettings>
+#include <fstream>
 
 #include <common/Formatting.h>
 #include <ffmpeg/AVCodecContextWrapper.h>
@@ -56,8 +57,18 @@ namespace
 
 auto startCode = QByteArrayLiteral("\x00\x00\x01");
 
+uint64_t getBoxSize(ByteVector::const_iterator iterator)
+{
+  uint64_t size = 0;
+  size += static_cast<uint64_t>(*(iterator++)) << (8 * 3);
+  size += static_cast<uint64_t>(*(iterator++)) << (8 * 2);
+  size += static_cast<uint64_t>(*(iterator++)) << (8 * 1);
+  size += static_cast<uint64_t>(*iterator);
+  return size;
 }
 
+} // namespace
+
 FileSourceFFmpegFile::FileSourceFFmpegFile()
 {
   connect(&this->fileWatcher,
@@ -217,6 +228,55 @@ StringPairVec FileSourceFFmpegFile::getMetadata()
   return ff.getDictionaryEntries(this->formatCtx.getMetadata(), "", 0);
 }
 
+ByteVector FileSourceFFmpegFile::getLhvCData()
+{
+  const auto inputFormat = this->formatCtx.getInputFormat();
+  const auto isMp4       = inputFormat.getName().contains("mp4");
+  if (!this->getVideoStreamCodecID().isHEVC() || !isMp4)
+    return {};
+
+  // This is a bit of a hack. The problem is that FFmpeg can currently not extract this for us.
+  // Maybe this will be added in the future. So the only option we have here is to manually extract
+  // the lhvC data from the mp4 file. In mp4, the boxes can be at the beginning or at the end of the
+  // file.
+  enum class SearchPosition
+  {
+    Beginning,
+    End
+  };
+
+  std::ifstream inputFile(this->fileName.toStdString(), std::ios::binary);
+  for (const auto searchPosition : {SearchPosition::Beginning, SearchPosition::End})
+  {
+    constexpr auto NR_SEARCH_BYTES = 5120;
+
+    if (searchPosition == SearchPosition::End)
+      inputFile.seekg(-NR_SEARCH_BYTES, std::ios_base::end);
+
+    const auto rawFileData = functions::readData(inputFile, NR_SEARCH_BYTES);
+    if (rawFileData.empty())
+      continue;
+
+    const std::string searchString = "lhvC";
+    auto              lhvcPos      = std::search(
+        rawFileData.begin(), rawFileData.end(), searchString.begin(), searchString.end());
+    if (lhvcPos == rawFileData.end())
+      continue;
+
+    if (std::distance(rawFileData.begin(), lhvcPos) < 4)
+      continue;
+
+    const auto boxSize = getBoxSize(lhvcPos - 4);
+    if (boxSize == 0 || boxSize > std::distance(lhvcPos, rawFileData.end()))
+      continue;
+
+    // We just return the payload without the box size or the "lhvC" tag
+    return ByteVector(lhvcPos + 4, lhvcPos + boxSize - 4);
+  }
+
+  return {};
+}
+
 QList<QByteArray> FileSourceFFmpegFile::getParameterSets()
 {
   if (!this->isFileOpened)
@@ -484,6 +544,7 @@ void FileSourceFFmpegFile::openFileAndFindVideoStream(QString fileName)
   if (!this->ff.openInput(this->formatCtx, fileName))
     return;
 
+  this->fileName = fileName;
   this->formatCtx.getInputFormat();
 
   for (unsigned idx = 0; idx < this->formatCtx.getNbStreams(); idx++)

YUViewLib/src/filesource/FileSourceFFmpegFile.h

@@ -35,6 +35,7 @@
 #include "FileSource.h"
 #include <ffmpeg/AVCodecIDWrapper.h>
 #include <ffmpeg/AVCodecParametersWrapper.h>
+#include <ffmpeg/AVInputFormatWrapper.h>
 #include <ffmpeg/AVPacketWrapper.h>
 #include <ffmpeg/FFmpegVersionHandler.h>
 #include <video/rgb/videoHandlerRGB.h>
@@ -85,6 +86,7 @@ class FileSourceFFmpegFile : public QObject
 
   QByteArray    getExtradata();
   StringPairVec getMetadata();
+  ByteVector    getLhvCData();
   // Return a list containing the raw data of all parameter set NAL units
   QList<QByteArray> getParameterSets();
 
@@ -135,6 +137,8 @@ private slots:
   FFmpeg::AVPacketWrapper        currentPacket;    //< A place for the curren (frame) input buffer
   bool                           endOfFile{false}; //< Are we at the end of file (draining mode)?
 
+  QString fileName;
+
   // Seek the stream to the given pts value, flush the decoder and load the first packet so
   // that we are ready to start decoding from this pts.
   int64_t                 duration{-1}; //< duration / AV_TIME_BASE is the duration in seconds

YUViewLib/src/parser/AVFormat/HVCC.cpp → YUViewLib/src/parser/AVFormat/DecoderConfigurationHvcC.cpp

@@ -30,57 +30,17 @@
  *   along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "HVCC.h"
+#include "DecoderConfigurationHvcC.h"
 
 namespace parser::avformat
 {
 
 using namespace reader;
 
-void HVCCNalUnit::parse(unsigned              unitID,
-                        SubByteReaderLogging &reader,
-                        ParserAnnexBHEVC *    hevcParser,
-                        BitratePlotModel *    bitrateModel)
-{
-  SubByteReaderLoggingSubLevel subLevel(reader, "nal unit " + std::to_string(unitID));
-
-  this->nalUnitLength = reader.readBits("nalUnitLength", 16);
-
-  // Get the bytes of the raw nal unit to pass to the "real" hevc parser
-  auto nalData = reader.readBytes("", nalUnitLength, Options().withLoggingDisabled());
-
-  // Let the hevc annexB parser parse this
-  auto parseResult =
-      hevcParser->parseAndAddNALUnit(unitID, nalData, {}, {}, reader.getCurrentItemTree());
-  if (parseResult.success && bitrateModel != nullptr && parseResult.bitrateEntry)
-    bitrateModel->addBitratePoint(0, *parseResult.bitrateEntry);
-}
-
-void HVCCNalArray::parse(unsigned              arrayID,
-                         SubByteReaderLogging &reader,
-                         ParserAnnexBHEVC *    hevcParser,
-                         BitratePlotModel *    bitrateModel)
-{
-  SubByteReaderLoggingSubLevel subLevel(reader, "nal unit array " + std::to_string(arrayID));
-
-  // The next 3 bytes contain info about the array
-  this->array_completeness = reader.readFlag("array_completeness");
-  reader.readFlag("reserved_flag_false", Options().withCheckEqualTo(0, CheckLevel::Warning));
-  this->nal_unit_type = reader.readBits("nal_unit_type", 6);
-  this->numNalus      = reader.readBits("numNalus", 16);
-
-  for (unsigned i = 0; i < numNalus; i++)
-  {
-    HVCCNalUnit nal;
-    nal.parse(i, reader, hevcParser, bitrateModel);
-    nalList.push_back(nal);
-  }
-}
-
-void HVCC::parse(ByteVector &              data,
-                 std::shared_ptr<TreeItem> root,
-                 ParserAnnexBHEVC *        hevcParser,
-                 BitratePlotModel *        bitrateModel)
+void DecoderConfigurationHvcC::parse(const ByteVector &        data,
+                                     std::shared_ptr<TreeItem> root,
+                                     ParserAnnexBHEVC *        hevcParser,
+                                     BitratePlotModel *        bitrateModel)
 {
   SubByteReaderLogging reader(data, root, "Extradata (HEVC hvcC format)");
   reader.disableEmulationPrevention();
@@ -123,10 +83,10 @@ void HVCC::parse(ByteVector &              data,
   // Now parse the contained raw NAL unit arrays
   for (unsigned i = 0; i < this->numOfArrays; i++)
   {
-    HVCCNalArray a;
-    a.parse(i, reader, hevcParser, bitrateModel);
-    this->naluArrays.push_back(a);
+    DecoderConfigurationNALArray array;
+    array.parse(i, reader, hevcParser, bitrateModel);
+    this->naluArrays.push_back(array);
   }
 }
 
-} // namespace parser::avformat
+} // namespace parser::avformat

YUViewLib/src/parser/AVFormat/HVCC.h → YUViewLib/src/parser/AVFormat/DecoderConfigurationHvcC.h

@@ -32,50 +32,21 @@
 
 #pragma once
 
-#include <common/Typedef.h>
 #include <parser/HEVC/ParserAnnexBHEVC.h>
 #include <parser/common/BitratePlotModel.h>
-#include <parser/common/SubByteReaderLogging.h>
 #include <parser/common/TreeItem.h>
 
-namespace parser::avformat
-{
+#include "DecoderConfigurationNalArray.h"
 
-class HVCCNalUnit
-{
-public:
-  HVCCNalUnit() = default;
-
-  void parse(unsigned                      unitID,
-             reader::SubByteReaderLogging &reader,
-             ParserAnnexBHEVC *            hevcParser,
-             BitratePlotModel *            bitrateModel);
-
-  unsigned nalUnitLength{};
-};
-
-class HVCCNalArray
+namespace parser::avformat
 {
-public:
-  HVCCNalArray() = default;
-
-  void parse(unsigned                      arrayID,
-             reader::SubByteReaderLogging &reader,
-             ParserAnnexBHEVC *            hevcParser,
-             BitratePlotModel *            bitrateModel);
-
-  bool                array_completeness{};
-  unsigned            nal_unit_type{};
-  unsigned            numNalus{};
-  vector<HVCCNalUnit> nalList;
-};
 
-class HVCC
+class DecoderConfigurationHvcC
 {
 public:
-  HVCC() = default;
+  DecoderConfigurationHvcC() = default;
 
-  void parse(ByteVector &              data,
+  void parse(const ByteVector &        data,
              std::shared_ptr<TreeItem> root,
              ParserAnnexBHEVC *        hevcParser,
              BitratePlotModel *        bitrateModel);
@@ -99,7 +70,7 @@ class HVCC
   unsigned lengthSizeMinusOne{};
   unsigned numOfArrays{};
 
-  vector<HVCCNalArray> naluArrays;
+  std::vector<DecoderConfigurationNALArray> naluArrays;
 };
 
-} // namespace parser::avformat
+} // namespace parser::avformat