[ntuple] improve type name renormalization

Add different code path for renormalizing a demangled typeid name. Such
type names come from RField<T>::TypeName(). In this case, the compiler
has done already most of the normalization work and we don't need to go
through ROOT meta. There are some parts ROOT meta does that we need to
repeat for this case, e.g. dropping optional template arguments of
stdlib containers.

Author: jblomer

tree/ntuple/inc/ROOT/RField.hxx

@@ -284,7 +284,7 @@ public:
 template <typename T, typename = void>
 class RField final : public RClassField {
 public:
-   static std::string TypeName() { return ROOT::Internal::GetRenormalizedDemangledTypeName(typeid(T)); }
+   static std::string TypeName() { return ROOT::Internal::GetRenormalizedTypeName(typeid(T)); }
    RField(std::string_view name) : RClassField(name, TypeName())
    {
       static_assert(std::is_class_v<T>, "no I/O support for this basic C++ type");

tree/ntuple/inc/ROOT/RField/RFieldProxiedCollection.hxx

@@ -265,7 +265,7 @@ struct IsCollectionProxy : HasCollectionProxyMemberType<T> {
 template <typename T>
 class RField<T, typename std::enable_if<IsCollectionProxy<T>::value>::type> final : public RProxiedCollectionField {
 public:
-   static std::string TypeName() { return ROOT::Internal::GetRenormalizedDemangledTypeName(typeid(T)); }
+   static std::string TypeName() { return ROOT::Internal::GetRenormalizedTypeName(typeid(T)); }
    RField(std::string_view name) : RProxiedCollectionField(name, TypeName())
    {
       static_assert(std::is_class<T>::value, "collection proxy unsupported for fundamental types");

tree/ntuple/inc/ROOT/RFieldUtils.hxx

@@ -27,7 +27,7 @@ std::string GetRenormalizedTypeName(const std::string &metaNormalizedName);
 
 /// Given a type info ask ROOT meta to demangle it, then renormalize the resulting type name for RNTuple. Useful to
 /// ensure that e.g. fundamental types are normalized to the type used by RNTuple (e.g. int -> std::int32_t).
-std::string GetRenormalizedDemangledTypeName(const std::type_info &ti);
+std::string GetRenormalizedTypeName(const std::type_info &ti);
 
 /// Applies all RNTuple type normalization rules except typedef resolution.
 std::string GetNormalizedUnresolvedTypeName(const std::string &origName);
@@ -57,8 +57,10 @@ std::tuple<std::string, std::vector<std::size_t>> ParseArrayType(const std::stri
 
 /// Used in RFieldBase::Create() in order to get the comma-separated list of template types
 /// E.g., gets {"int", "std::variant<double,int>"} from "int,std::variant<double,int>".
+/// If maxArgs > 0, stop tokenizing after the given number of tokens are found. Used to strip
+/// STL allocator and other optional arguments.
 /// TODO(jblomer): Try to merge with TClassEdit::TSplitType
-std::vector<std::string> TokenizeTypeList(std::string_view templateType);
+std::vector<std::string> TokenizeTypeList(std::string_view templateType, std::size_t maxArgs = 0);
 
 } // namespace Internal
 } // namespace ROOT

tree/ntuple/inc/ROOT/RNTupleReader.hxx

@@ -324,7 +324,7 @@ public:
    /// \sa GetView(std::string_view, std::shared_ptr<T>)
    ROOT::RNTupleView<void> GetView(std::string_view fieldName, void *rawPtr, const std::type_info &ti)
    {
-      return GetView(RetrieveFieldId(fieldName), rawPtr, ROOT::Internal::GetRenormalizedDemangledTypeName(ti));
+      return GetView(RetrieveFieldId(fieldName), rawPtr, ROOT::Internal::GetRenormalizedTypeName(ti));
    }
 
    /// Provides access to an individual (sub)field from its on-disk ID.
@@ -377,7 +377,7 @@ public:
    /// \sa GetView(std::string_view, std::shared_ptr<T>)
    ROOT::RNTupleView<void> GetView(ROOT::DescriptorId_t fieldId, void *rawPtr, const std::type_info &ti)
    {
-      return GetView(fieldId, rawPtr, ROOT::Internal::GetRenormalizedDemangledTypeName(ti));
+      return GetView(fieldId, rawPtr, ROOT::Internal::GetRenormalizedTypeName(ti));
    }
 
    /// Provides direct access to the I/O buffers of a **mappable** (sub)field.

tree/ntuple/src/RFieldUtils.cxx

@@ -118,25 +118,14 @@ std::vector<AnglePos> FindTemplateAngleBrackets(const std::string &typeName)
    return result;
 }
 
-} // namespace
-
-std::string ROOT::Internal::GetCanonicalTypePrefix(const std::string &typeName)
+// TClassEdit::CleanType and the name demangling insert blanks between closing angle brackets,
+// as they were required before C++11. We want to remove them for RNTuple.
+void RemoveSpaceBeforeClosingAngleBracket(std::string &typeName)
 {
-   std::string canonicalType{TClassEdit::CleanType(typeName.c_str(), /*mode=*/1)};
-   if (canonicalType.substr(0, 7) == "struct ") {
-      canonicalType.erase(0, 7);
-   } else if (canonicalType.substr(0, 5) == "enum ") {
-      canonicalType.erase(0, 5);
-   } else if (canonicalType.substr(0, 2) == "::") {
-      canonicalType.erase(0, 2);
-   }
-
-   // TClassEdit::CleanType inserts blanks between closing angle brackets, as they were required before C++11. We want
-   // to remove them for RNTuple.
-   auto angle = canonicalType.find('<');
+   auto angle = typeName.find('<');
    if (angle != std::string::npos) {
-      auto dst = canonicalType.begin() + angle;
-      auto end = canonicalType.end();
+      auto dst = typeName.begin() + angle;
+      auto end = typeName.end();
       for (auto src = dst; src != end; ++src) {
          if (*src == ' ') {
             auto next = src + 1;
@@ -147,9 +136,185 @@ std::string ROOT::Internal::GetCanonicalTypePrefix(const std::string &typeName)
          }
          *(dst++) = *src;
       }
-      canonicalType.erase(dst, end);
+      typeName.erase(dst, end);
+   }
+}
+
+// The demangled name adds spaces after commas
+void RemoveSpaceAfterComma(std::string &typeName)
+{
+   auto itr = typeName.begin();
+   while (itr != typeName.end()) {
+      auto c = *itr;
+      itr++;
+
+      if (c != ',')
+         continue;
+
+      R__ASSERT(itr != typeName.end());
+      if (*itr == ' ') {
+         itr = typeName.erase(itr);
+      }
+   }
+}
+
+// Map fundamental integer types to stdint integer types (e.g. int --> std::int32_t)
+void MapIntegerType(std::string &typeName)
+{
+   if (typeName == "signed char") {
+      typeName = ROOT::RField<signed char>::TypeName();
+   } else if (typeName == "unsigned char") {
+      typeName = ROOT::RField<unsigned char>::TypeName();
+   } else if (typeName == "short" || typeName == "short int" || typeName == "signed short" ||
+              typeName == "signed short int") {
+      typeName = ROOT::RField<short int>::TypeName();
+   } else if (typeName == "unsigned short" || typeName == "unsigned short int") {
+      typeName = ROOT::RField<unsigned short int>::TypeName();
+   } else if (typeName == "int" || typeName == "signed" || typeName == "signed int") {
+      typeName = ROOT::RField<int>::TypeName();
+   } else if (typeName == "unsigned" || typeName == "unsigned int") {
+      typeName = ROOT::RField<unsigned int>::TypeName();
+   } else if (typeName == "long" || typeName == "long int" || typeName == "signed long" ||
+              typeName == "signed long int") {
+      typeName = ROOT::RField<long int>::TypeName();
+   } else if (typeName == "unsigned long" || typeName == "unsigned long int") {
+      typeName = ROOT::RField<unsigned long int>::TypeName();
+   } else if (typeName == "long long" || typeName == "long long int" || typeName == "signed long long" ||
+              typeName == "signed long long int") {
+      typeName = ROOT::RField<long long int>::TypeName();
+   } else if (typeName == "unsigned long long" || typeName == "unsigned long long int") {
+      typeName = ROOT::RField<unsigned long long int>::TypeName();
+   }
+}
+
+std::string GetRenormalizedMetaTypeName(const std::string &metaNormalizedName)
+{
+   const std::string canonicalTypePrefix{ROOT::Internal::GetCanonicalTypePrefix(metaNormalizedName)};
+   // RNTuple resolves Double32_t for the normalized type name but keeps Double32_t for the type alias
+   // (also in template parameters)
+   if (canonicalTypePrefix == "Double32_t")
+      return "double";
+
+   if (canonicalTypePrefix.find('<') == std::string::npos) {
+      // If there are no templates, the function is done.
+      return canonicalTypePrefix;
+   }
+
+   const auto angleBrackets = FindTemplateAngleBrackets(canonicalTypePrefix);
+   R__ASSERT(!angleBrackets.empty());
+
+   std::string normName;
+   std::string::size_type currentPos = 0;
+   for (std::size_t i = 0; i < angleBrackets.size(); i++) {
+      const auto [posOpen, posClose] = angleBrackets[i];
+      // Append the type prefix until the open angle bracket.
+      normName += canonicalTypePrefix.substr(currentPos, posOpen + 1 - currentPos);
+
+      const auto argList = canonicalTypePrefix.substr(posOpen + 1, posClose - posOpen - 1);
+      const auto templateArgs = ROOT::Internal::TokenizeTypeList(argList);
+      R__ASSERT(!templateArgs.empty());
+
+      for (const auto &a : templateArgs) {
+         normName += GetNormalizedTemplateArg(a, GetRenormalizedMetaTypeName) + ",";
+      }
+
+      normName[normName.size() - 1] = '>';
+      currentPos = posClose + 1;
+   }
+
+   // Append the rest of the type from the last closing angle bracket.
+   const auto lastClosePos = angleBrackets.back().second;
+   normName += canonicalTypePrefix.substr(lastClosePos + 1);
+
+   return normName;
+}
+
+std::string GetRenormalizedDemangledTypeName(const std::string &demangledName)
+{
+   std::string canonicalTypePrefix{demangledName};
+   MapIntegerType(canonicalTypePrefix);
+
+   if (canonicalTypePrefix.find('<') == std::string::npos) {
+      // If there are no templates, the function is done.
+      return canonicalTypePrefix;
+   }
+   RemoveSpaceBeforeClosingAngleBracket(canonicalTypePrefix);
+   RemoveSpaceAfterComma(canonicalTypePrefix);
+
+   const auto angleBrackets = FindTemplateAngleBrackets(canonicalTypePrefix);
+   R__ASSERT(!angleBrackets.empty());
+
+   // Remove optional stdlib template arguments
+   int maxTemplateArgs = 0;
+   if (canonicalTypePrefix.rfind("std::vector<", 0) == 0  || canonicalTypePrefix.rfind("std::set<", 0) == 0 ||
+       canonicalTypePrefix.rfind("std::unordered_set<", 0) == 0 ||
+       canonicalTypePrefix.rfind("std::multiset<", 0) == 0 ||
+       canonicalTypePrefix.rfind("std::unordered_multiset<", 0) == 0 ||
+       canonicalTypePrefix.rfind("std::unique_ptr<", 0) == 0) {
+      maxTemplateArgs = 1;
+   } else if (canonicalTypePrefix.rfind("std::map<", 0) == 0 ||
+              canonicalTypePrefix.rfind("std::unordered_map<", 0) == 0 ||
+              canonicalTypePrefix.rfind("std::multimap<", 0) == 0 ||
+              canonicalTypePrefix.rfind("std::unordered_multimap<", 0) == 0) {
+     maxTemplateArgs = 2;
+   }
+
+   std::string normName;
+   std::string::size_type currentPos = 0;
+   for (std::size_t i = 0; i < angleBrackets.size(); i++) {
+      const auto [posOpen, posClose] = angleBrackets[i];
+      // Append the type prefix until the open angle bracket.
+      normName += canonicalTypePrefix.substr(currentPos, posOpen + 1 - currentPos);
+
+      const auto argList = canonicalTypePrefix.substr(posOpen + 1, posClose - posOpen - 1);
+      auto templateArgs = ROOT::Internal::TokenizeTypeList(argList, maxTemplateArgs);
+      R__ASSERT(!templateArgs.empty());
+
+      for (const auto &a : templateArgs) {
+         normName += GetNormalizedTemplateArg(a, GetRenormalizedDemangledTypeName) + ",";
+      }
+
+      normName[normName.size() - 1] = '>';
+      currentPos = posClose + 1;
+   }
+
+   // Append the rest of the type from the last closing angle bracket.
+   const auto lastClosePos = angleBrackets.back().second;
+   normName += canonicalTypePrefix.substr(lastClosePos + 1);
+
+   // Reverse std::string --> std::basic_string<char> typedef
+   static std::string stringName = []() {
+      int e;
+      char *str = TClassEdit::DemangleName(typeid(std::string).name(), e);
+      R__ASSERT(str && e == 0);
+      std::string res{str};
+      free(str);
+      res.erase(std::remove(res.begin(), res.end(), ' '), res.end());
+      return res;
+   }();
+   if (normName == stringName)
+      return "std::string";
+
+   return normName;
+}
+
+} // namespace
+
+std::string ROOT::Internal::GetCanonicalTypePrefix(const std::string &typeName)
+{
+   // Remove outer cv qualifiers
+   std::string canonicalType{TClassEdit::CleanType(typeName.c_str(), /*mode=*/1)};
+
+   if (canonicalType.substr(0, 7) == "struct ") {
+      canonicalType.erase(0, 7);
+   } else if (canonicalType.substr(0, 5) == "enum ") {
+      canonicalType.erase(0, 5);
+   } else if (canonicalType.substr(0, 2) == "::") {
+      canonicalType.erase(0, 2);
    }
 
+   RemoveSpaceBeforeClosingAngleBracket(canonicalType);
+
    if (canonicalType.substr(0, 6) == "array<") {
       canonicalType = "std::" + canonicalType;
    } else if (canonicalType.substr(0, 7) == "atomic<") {
@@ -191,77 +356,27 @@ std::string ROOT::Internal::GetCanonicalTypePrefix(const std::string &typeName)
       canonicalType = it->second;
    }
 
-   // Map fundamental integer types to stdint integer types (e.g. int --> std::int32_t)
-   if (canonicalType == "signed char") {
-      canonicalType = RField<signed char>::TypeName();
-   } else if (canonicalType == "unsigned char") {
-      canonicalType = RField<unsigned char>::TypeName();
-   } else if (canonicalType == "short" || canonicalType == "short int" || canonicalType == "signed short" ||
-              canonicalType == "signed short int") {
-      canonicalType = RField<short int>::TypeName();
-   } else if (canonicalType == "unsigned short" || canonicalType == "unsigned short int") {
-      canonicalType = RField<unsigned short int>::TypeName();
-   } else if (canonicalType == "int" || canonicalType == "signed" || canonicalType == "signed int") {
-      canonicalType = RField<int>::TypeName();
-   } else if (canonicalType == "unsigned" || canonicalType == "unsigned int") {
-      canonicalType = RField<unsigned int>::TypeName();
-   } else if (canonicalType == "long" || canonicalType == "long int" || canonicalType == "signed long" ||
-              canonicalType == "signed long int") {
-      canonicalType = RField<long int>::TypeName();
-   } else if (canonicalType == "unsigned long" || canonicalType == "unsigned long int") {
-      canonicalType = RField<unsigned long int>::TypeName();
-   } else if (canonicalType == "long long" || canonicalType == "long long int" || canonicalType == "signed long long" ||
-              canonicalType == "signed long long int") {
-      canonicalType = RField<long long int>::TypeName();
-   } else if (canonicalType == "unsigned long long" || canonicalType == "unsigned long long int") {
-      canonicalType = RField<unsigned long long int>::TypeName();
-   }
+   MapIntegerType(canonicalType);
 
    return canonicalType;
 }
 
-std::string ROOT::Internal::GetRenormalizedTypeName(const std::string &metaNormalizedName)
+std::string ROOT::Internal::GetRenormalizedTypeName(const std::type_info &ti)
 {
-   const std::string canonicalTypePrefix{GetCanonicalTypePrefix(metaNormalizedName)};
-   // RNTuple resolves Double32_t for the normalized type name but keeps Double32_t for the type alias
-   // (also in template parameters)
-   if (canonicalTypePrefix == "Double32_t")
-      return "double";
-
-   if (canonicalTypePrefix.find('<') == std::string::npos) {
-      // If there are no templates, the function is done.
-      return canonicalTypePrefix;
-   }
-
-   const auto angleBrackets = FindTemplateAngleBrackets(canonicalTypePrefix);
-   R__ASSERT(!angleBrackets.empty());
-
-   std::string normName;
-   std::string::size_type currentPos = 0;
-   for (std::size_t i = 0; i < angleBrackets.size(); i++) {
-      const auto [posOpen, posClose] = angleBrackets[i];
-      // Append the type prefix until the open angle bracket.
-      normName += canonicalTypePrefix.substr(currentPos, posOpen + 1 - currentPos);
-
-      const auto argList = canonicalTypePrefix.substr(posOpen + 1, posClose - posOpen - 1);
-      const auto templateArgs = TokenizeTypeList(argList);
-      R__ASSERT(!templateArgs.empty());
-
-      for (const auto &a : templateArgs) {
-         normName += GetNormalizedTemplateArg(a, GetRenormalizedTypeName) + ",";
-      }
-
-      normName[normName.size() - 1] = '>';
-      currentPos = posClose + 1;
-   }
-
-   // Append the rest of the type from the last closing angle bracket.
-   const auto lastClosePos = angleBrackets.back().second;
-   normName += canonicalTypePrefix.substr(lastClosePos + 1);
+   int errCode;
+   char *cstrDemangledName = TClassEdit::DemangleName(ti.name(), errCode);
+   R__ASSERT(cstrDemangledName && errCode == 0);
+   std::string normName{GetRenormalizedDemangledTypeName(cstrDemangledName)};
+   free(cstrDemangledName);
 
    return normName;
 }
 
+std::string ROOT::Internal::GetRenormalizedTypeName(const std::string &metaNormalizedName)
+{
+   return GetRenormalizedMetaTypeName(metaNormalizedName);
+}
+
 std::string ROOT::Internal::GetNormalizedUnresolvedTypeName(const std::string &origName)
 {
    const TClassEdit::EModType modType = static_cast<TClassEdit::EModType>(
@@ -435,7 +550,7 @@ std::tuple<std::string, std::vector<std::size_t>> ROOT::Internal::ParseArrayType
    return std::make_tuple(prefix, sizeVec);
 }
 
-std::vector<std::string> ROOT::Internal::TokenizeTypeList(std::string_view templateType)
+std::vector<std::string> ROOT::Internal::TokenizeTypeList(std::string_view templateType, std::size_t maxArgs)
 {
    std::vector<std::string> result;
    if (templateType.empty())
@@ -452,6 +567,8 @@ std::vector<std::string> ROOT::Internal::TokenizeTypeList(std::string_view templ
       case ',':
          if (nestingLevel == 0) {
             result.push_back(std::string(typeBegin, typeCursor - typeBegin));
+            if (maxArgs && result.size() == maxArgs)
+               return result;
             typeBegin = typeCursor + 1;
          }
          break;
@@ -461,8 +578,3 @@ std::vector<std::string> ROOT::Internal::TokenizeTypeList(std::string_view templ
    result.push_back(std::string(typeBegin, typeCursor - typeBegin));
    return result;
 }
-
-std::string ROOT::Internal::GetRenormalizedDemangledTypeName(const std::type_info &ti)
-{
-   return ROOT::Internal::GetRenormalizedTypeName(ROOT::Internal::GetDemangledTypeName(ti));
-}