From a37dc49554f474bd494c49865d14f23ad00cc673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Dallaire=20C=C3=B4t=C3=A9?= <110583667+0mdc@users.noreply.github.com> Date: Wed, 1 May 2024 21:24:09 -0700 Subject: [PATCH] Add instance metadata to gfx-replay. (#2377) * Add instance metadata to gfx-replay. * Update metadata like state updates rather than creations. * Fix json parsing. --- src/esp/gfx/replay/Keyframe.h | 17 +++++++++++--- src/esp/gfx/replay/Player.cpp | 39 +++++++++++++++++++++++---------- src/esp/gfx/replay/Player.h | 17 +++++++++----- src/esp/gfx/replay/Recorder.cpp | 24 +++++++++++++++----- src/esp/gfx/replay/Recorder.h | 2 ++ src/esp/io/JsonEspTypes.cpp | 25 +++++++++++++++++++++ src/esp/io/JsonEspTypes.h | 18 +++++++++++++-- src/tests/GfxReplayTest.cpp | 35 +++++++++++++++++++++++------ src/tests/IOTest.cpp | 2 +- 9 files changed, 143 insertions(+), 36 deletions(-) diff --git a/src/esp/gfx/replay/Keyframe.h b/src/esp/gfx/replay/Keyframe.h index 92a85233bf..d7cf4c6bee 100644 --- a/src/esp/gfx/replay/Keyframe.h +++ b/src/esp/gfx/replay/Keyframe.h @@ -31,11 +31,21 @@ struct Transform { */ struct RenderAssetInstanceState { Transform absTransform; // localToWorld - // note we currently only support semanticId per instance, not per drawable - int semanticId = ID_UNDEFINED; // note we don't currently support runtime changes to lightSetupKey bool operator==(const RenderAssetInstanceState& rhs) const { - return absTransform == rhs.absTransform && semanticId == rhs.semanticId; + return absTransform == rhs.absTransform; + } +}; + +/** + * @brief Metadata associated with an instance. + */ +struct InstanceMetadata { + int objectId = ID_UNDEFINED; + int semanticId = ID_UNDEFINED; + + bool operator==(const InstanceMetadata& rhs) const { + return objectId == rhs.objectId && semanticId == rhs.semanticId; } }; @@ -67,6 +77,7 @@ struct Keyframe { esp::assets::RenderAssetInstanceCreationInfo>> creations; std::vector deletions; + std::vector> metadata; std::vector> stateUpdates; std::vector rigUpdates; diff --git a/src/esp/gfx/replay/Player.cpp b/src/esp/gfx/replay/Player.cpp index 4c8774c351..d4335f2af4 100644 --- a/src/esp/gfx/replay/Player.cpp +++ b/src/esp/gfx/replay/Player.cpp @@ -36,7 +36,8 @@ std::string removeMaterialOverrideFromFilepathAndWarn(const std::string& src) { static_assert(std::is_nothrow_move_constructible::value, ""); -void AbstractPlayerImplementation::setNodeSemanticId(NodeHandle, unsigned) {} +void AbstractPlayerImplementation::setNodeMetadata(NodeHandle, + const InstanceMetadata&) {} void AbstractPlayerImplementation::changeLightSetup(const LightSetup&) {} @@ -75,10 +76,15 @@ Mn::Matrix4 AbstractSceneGraphPlayerImplementation::hackGetNodeTransform( return (*reinterpret_cast(node)).transformation(); } -void AbstractSceneGraphPlayerImplementation::setNodeSemanticId( +void AbstractSceneGraphPlayerImplementation::setNodeMetadata( const NodeHandle node, - const unsigned id) { - setSemanticIdForSubtree(reinterpret_cast(node), id); + const InstanceMetadata& metadata) { + setSemanticInfoForSubtree(reinterpret_cast(node), + { + metadata.semanticId, + metadata.objectId, + ID_UNDEFINED, + }); } void AbstractPlayerImplementation::createRigInstance( @@ -186,7 +192,7 @@ void Player::clearFrame() { implementation_->deleteAssetInstances(createdInstances_); createdInstances_.clear(); assetInfos_.clear(); - creationInfos_.clear(); + creationRecords_.clear(); frameIndex_ = -1; } @@ -235,7 +241,16 @@ void Player::applyKeyframe(const Keyframe& keyframe) { const auto& instanceKey = pair.first; CORRADE_INTERNAL_ASSERT(createdInstances_.count(instanceKey) == 0); createdInstances_[instanceKey] = node; - creationInfos_[instanceKey] = adjustedCreation; + + creationRecords_[instanceKey] = + CreationRecord{adjustedCreation, InstanceMetadata()}; + } + + for (const auto& pair : keyframe.metadata) { + const auto& instanceKey = pair.first; + implementation_->setNodeMetadata(createdInstances_[instanceKey], + pair.second); + creationRecords_[instanceKey].metadata = pair.second; } hackProcessDeletions(keyframe); @@ -250,7 +265,6 @@ void Player::applyKeyframe(const Keyframe& keyframe) { const auto& state = pair.second; implementation_->setNodeTransform(node, state.absTransform.translation, state.absTransform.rotation); - implementation_->setNodeSemanticId(node, state.semanticId); } for (const auto& rigUpdate : keyframe.rigUpdates) { @@ -282,11 +296,11 @@ void Player::hackProcessDeletions(const Keyframe& keyframe) { implementation_->deleteAssetInstance(it->second); createdInstances_.erase(deletionInstanceKey); - int rigId = creationInfos_[deletionInstanceKey].rigId; + int rigId = creationRecords_[deletionInstanceKey].creationInfo.rigId; if (rigId != ID_UNDEFINED) { implementation_->deleteRigInstance(rigId); } - creationInfos_.erase(deletionInstanceKey); + creationRecords_.erase(deletionInstanceKey); } } else if (keyframe.deletions.size() > 0) { // Cache latest transforms @@ -308,14 +322,17 @@ void Player::hackProcessDeletions(const Keyframe& keyframe) { continue; } createdInstances_.erase(createInstanceIt); - creationInfos_.erase(deletion); + creationRecords_.erase(deletion); } for (const auto& pair : createdInstances_) { const auto key = pair.first; - const auto& creationInfo = creationInfos_[key]; + const auto& creationRecord = creationRecords_[key]; + const auto& creationInfo = creationRecord.creationInfo; auto* instance = implementation_->loadAndCreateRenderAssetInstance( assetInfos_[creationInfo.filepath], creationInfo); + const auto& metadata = creationRecord.metadata; + implementation_->setNodeMetadata(instance, metadata); // Replace dangling reference createdInstances_[key] = instance; diff --git a/src/esp/gfx/replay/Player.h b/src/esp/gfx/replay/Player.h index 0087f2f356..3bbd39f5fb 100644 --- a/src/esp/gfx/replay/Player.h +++ b/src/esp/gfx/replay/Player.h @@ -18,6 +18,11 @@ namespace replay { class Player; +struct CreationRecord { + assets::RenderAssetInstanceCreationInfo creationInfo; + InstanceMetadata metadata; +}; + /** @brief Node handle @@ -112,13 +117,14 @@ class AbstractPlayerImplementation { virtual Mn::Matrix4 hackGetNodeTransform(NodeHandle node) const = 0; /** - * @brief Set node semantic ID + * @brief Set node metadata. * * The @p handle is expected to be returned from an earlier call to * @ref loadAndCreateRenderAssetInstance() on the same instance. Default * implementation does nothing. */ - virtual void setNodeSemanticId(NodeHandle node, unsigned id); + virtual void setNodeMetadata(NodeHandle node, + const InstanceMetadata& metadata); /** * @brief Change light setup @@ -176,7 +182,8 @@ class AbstractSceneGraphPlayerImplementation Mn::Matrix4 hackGetNodeTransform(NodeHandle node) const override; - void setNodeSemanticId(NodeHandle node, unsigned id) override; + void setNodeMetadata(NodeHandle node, + const InstanceMetadata& metadata) override; }; /** @@ -304,9 +311,7 @@ class Player { std::vector keyframes_; std::unordered_map assetInfos_; std::unordered_map createdInstances_; - std::unordered_map - creationInfos_; + std::unordered_map creationRecords_; std::unordered_map latestTransformCache_; std::set failedFilepaths_; diff --git a/src/esp/gfx/replay/Recorder.cpp b/src/esp/gfx/replay/Recorder.cpp index c7055b90f9..5d0003e6f0 100644 --- a/src/esp/gfx/replay/Recorder.cpp +++ b/src/esp/gfx/replay/Recorder.cpp @@ -81,9 +81,9 @@ void Recorder::onCreateRenderAssetInstance( // manually later if necessary. NodeDeletionHelper* deletionHelper = new NodeDeletionHelper{*node, this}; - instanceRecords_.emplace_back(InstanceRecord{node, instanceKey, - Corrade::Containers::NullOpt, - deletionHelper, creation.rigId}); + instanceRecords_.emplace_back(InstanceRecord{ + node, instanceKey, Corrade::Containers::NullOpt, + Corrade::Containers::NullOpt, deletionHelper, creation.rigId}); } void Recorder::onCreateRigInstance(int rigId, const Rig& rig) { @@ -214,7 +214,13 @@ RenderAssetInstanceState Recorder::getInstanceState( const scene::SceneNode* node) { const auto absTransformMat = node->absoluteTransformation(); const auto transform = ::createReplayTransform(absTransformMat); - return RenderAssetInstanceState{transform, node->getSemanticId()}; + return RenderAssetInstanceState{transform}; +} + +InstanceMetadata Recorder::getInstanceMetadata(const scene::SceneNode* node) { + const auto objectId = node->getBaseObjectId(); + const auto semanticId = node->getSemanticId(); + return InstanceMetadata{objectId, semanticId}; } void Recorder::updateStates() { @@ -230,6 +236,11 @@ void Recorder::updateInstanceStates() { state); instanceRecord.recentState = state; } + auto metadata = getInstanceMetadata(instanceRecord.node); + if (!instanceRecord.metadata || metadata != instanceRecord.metadata) { + getKeyframe().metadata.emplace_back(instanceRecord.instanceKey, metadata); + instanceRecord.metadata = metadata; + } } } @@ -332,10 +343,11 @@ void Recorder::consolidateSavedKeyframes() { // consolidate saved keyframes into current keyframe addLoadsCreationsDeletions(savedKeyframes_.begin(), savedKeyframes_.end(), &getKeyframe()); - // clear instanceRecord.recentState to ensure updates get included in the next - // saved keyframe. + // clear instanceRecord.recentState and metadata to ensure updates get + // included in the next saved keyframe. for (auto& instanceRecord : instanceRecords_) { instanceRecord.recentState = Corrade::Containers::NullOpt; + instanceRecord.metadata = Corrade::Containers::NullOpt; } savedKeyframes_.clear(); } diff --git a/src/esp/gfx/replay/Recorder.h b/src/esp/gfx/replay/Recorder.h index 47d8c3cc1c..3fc5477df0 100644 --- a/src/esp/gfx/replay/Recorder.h +++ b/src/esp/gfx/replay/Recorder.h @@ -185,6 +185,7 @@ class Recorder { scene::SceneNode* node = nullptr; RenderAssetInstanceKey instanceKey = ID_UNDEFINED; Corrade::Containers::Optional recentState; + Corrade::Containers::Optional metadata; NodeDeletionHelper* deletionHelper = nullptr; int rigId = ID_UNDEFINED; }; @@ -198,6 +199,7 @@ class Recorder { RenderAssetInstanceKey getNewInstanceKey(); int findInstance(const scene::SceneNode* queryNode); RenderAssetInstanceState getInstanceState(const scene::SceneNode* node); + InstanceMetadata getInstanceMetadata(const scene::SceneNode* node); void updateStates(); void updateInstanceStates(); void updateRigInstanceStates(); diff --git a/src/esp/io/JsonEspTypes.cpp b/src/esp/io/JsonEspTypes.cpp index 8296645697..2736004241 100644 --- a/src/esp/io/JsonEspTypes.cpp +++ b/src/esp/io/JsonEspTypes.cpp @@ -49,6 +49,17 @@ JsonGenericValue toJsonValue(const gfx::replay::Keyframe& keyframe, io::addMember(obj, "stateUpdates", stateUpdatesArray, allocator); } + if (!keyframe.metadata.empty()) { + JsonGenericValue metadataArray(rapidjson::kArrayType); + for (const auto& pair : keyframe.metadata) { + JsonGenericValue metadataObj(rapidjson::kObjectType); + io::addMember(metadataObj, "instanceKey", pair.first, allocator); + io::addMember(metadataObj, "metadata", pair.second, allocator); + metadataArray.PushBack(metadataObj, allocator); + } + io::addMember(obj, "metadata", metadataArray, allocator); + } + if (!keyframe.rigUpdates.empty()) { JsonGenericValue rigUpdatesArray(rapidjson::kArrayType); for (const auto& rig : keyframe.rigUpdates) { @@ -132,6 +143,20 @@ bool fromJsonValue(const JsonGenericValue& obj, } } + itr = obj.FindMember("metadata"); + if (itr != obj.MemberEnd()) { + const JsonGenericValue& metadataArray = itr->value; + keyframe.metadata.reserve(metadataArray.Size()); + for (const auto& metadataObj : metadataArray.GetArray()) { + std::pair + pair; + io::readMember(metadataObj, "instanceKey", pair.first); + io::readMember(metadataObj, "metadata", pair.second); + keyframe.metadata.emplace_back(std::move(pair)); + } + } + itr = obj.FindMember("rigUpdates"); if (itr != obj.MemberEnd()) { const JsonGenericValue& rigUpdatesArray = itr->value; diff --git a/src/esp/io/JsonEspTypes.h b/src/esp/io/JsonEspTypes.h index 4ecc36e94b..3a854bfc39 100644 --- a/src/esp/io/JsonEspTypes.h +++ b/src/esp/io/JsonEspTypes.h @@ -140,6 +140,22 @@ inline bool fromJsonValue(const JsonGenericValue& obj, return true; } +inline JsonGenericValue toJsonValue(const esp::gfx::replay::InstanceMetadata& x, + JsonAllocator& allocator) { + JsonGenericValue obj(rapidjson::kObjectType); + addMember(obj, "objectId", x.objectId, allocator); + addMember(obj, "semanticId", x.semanticId, allocator); + return obj; +} + +inline bool fromJsonValue(const JsonGenericValue& obj, + esp::gfx::replay::InstanceMetadata& x) { + bool success = true; + success &= readMember(obj, "objectId", x.objectId); + success &= readMember(obj, "semanticId", x.semanticId); + return success; +} + inline JsonGenericValue toJsonValue(const esp::gfx::replay::Transform& x, JsonAllocator& allocator) { JsonGenericValue obj(rapidjson::kObjectType); @@ -161,14 +177,12 @@ inline JsonGenericValue toJsonValue( JsonAllocator& allocator) { JsonGenericValue obj(rapidjson::kObjectType); addMember(obj, "absTransform", x.absTransform, allocator); - addMember(obj, "semanticId", x.semanticId, allocator); return obj; } inline bool fromJsonValue(const JsonGenericValue& obj, esp::gfx::replay::RenderAssetInstanceState& x) { readMember(obj, "absTransform", x.absTransform); - readMember(obj, "semanticId", x.semanticId); return true; } diff --git a/src/tests/GfxReplayTest.cpp b/src/tests/GfxReplayTest.cpp index d453a62098..4797f8dda4 100644 --- a/src/tests/GfxReplayTest.cpp +++ b/src/tests/GfxReplayTest.cpp @@ -125,13 +125,16 @@ void GfxReplayTest::testRecorder() { info2.overridePhongMaterial->ambientColor = Mn::Color4(0.1, 0.2, 0.3, 0.4); info2.overridePhongMaterial->diffuseColor = Mn::Color4(0.2, 0.3, 0.4, 0.5); info2.overridePhongMaterial->specularColor = Mn::Color4(0.3, 0.4, 0.5, 0.6); + node->setSemanticId(7); + node->setBaseObjectId(9); esp::gfx::replay::Recorder recorder; recorder.onLoadRenderAsset(info); recorder.onCreateRenderAssetInstance(node, creation); recorder.saveKeyframe(); + + node->setSemanticId(5); node->setTranslation(Mn::Vector3(1.f, 2.f, 3.f)); - node->setSemanticId(7); // add the new override AssetInfo after 1st keyframe auto* node2 = resourceManager.loadAndCreateRenderAssetInstance( @@ -162,13 +165,17 @@ void GfxReplayTest::testRecorder() { esp::gfx::replay::RenderAssetInstanceKey instanceKey = keyframes[0].creations[0].first; CORRADE_COMPARE(keyframes[0].stateUpdates[0].first, instanceKey); + CORRADE_COMPARE(keyframes[0].metadata.size(), 1); + CORRADE_COMPARE(keyframes[0].metadata[0].second.semanticId, 7); + CORRADE_COMPARE(keyframes[0].metadata[0].second.objectId, 9); // verify frame #1 has an updated state for node and state for new node2 CORRADE_COMPARE(keyframes[1].stateUpdates.size(), 2); + CORRADE_COMPARE(keyframes[1].metadata.size(), 2); // verify frame #1 has our translation and semantic Id CORRADE_COMPARE(keyframes[1].stateUpdates[0].second.absTransform.translation, Mn::Vector3(1.f, 2.f, 3.f)); - CORRADE_COMPARE(keyframes[1].stateUpdates[0].second.semanticId, 7); + CORRADE_COMPARE(keyframes[1].metadata[0].second.semanticId, 5); // verify override material AssetInfo is loaded correctly CORRADE_COMPARE(keyframes[1].loads.size(), 1); @@ -256,6 +263,8 @@ void GfxReplayTest::testPlayer() { flags |= esp::assets::RenderAssetInstanceCreationInfo::Flag::IsSemantic; esp::assets::RenderAssetInstanceCreationInfo creation( boxFile, Corrade::Containers::NullOpt, flags, lightSetupKey); + constexpr int semanticId = 11; + constexpr int objectId = 22; /* // Keyframe struct shown here for reference @@ -276,14 +285,23 @@ void GfxReplayTest::testPlayer() { keyframes.emplace_back(esp::gfx::replay::Keyframe{ {info}, {}, {{instanceKey, creation}}, {}, {}, {}, {}}); - constexpr int semanticId = 4; + esp::gfx::replay::InstanceMetadata instanceMetadata{}; + instanceMetadata.objectId = objectId; + instanceMetadata.semanticId = semanticId; + esp::gfx::replay::RenderAssetInstanceState stateUpdate{ - {Mn::Vector3(1.f, 2.f, 3.f), Mn::Quaternion(Mn::Math::IdentityInit)}, - semanticId}; + {Mn::Vector3(1.f, 2.f, 3.f), Mn::Quaternion(Mn::Math::IdentityInit)}}; // keyframe #1: a state update - keyframes.emplace_back(esp::gfx::replay::Keyframe{ - {}, {}, {}, {}, {{instanceKey, stateUpdate}}, {}, {}}); + keyframes.emplace_back( + esp::gfx::replay::Keyframe{{}, + {}, + {}, + {}, + {{instanceKey, instanceMetadata}}, + {{instanceKey, stateUpdate}}, + {}, + {}}); // keyframe #2: delete instance keyframes.emplace_back( @@ -297,6 +315,7 @@ void GfxReplayTest::testPlayer() { {}, {}, {}, + {}, {{"my_user_transform", {Mn::Vector3(4.f, 5.f, 6.f), Mn::Quaternion(Mn::Math::IdentityInit)}}}}); @@ -336,6 +355,7 @@ void GfxReplayTest::testPlayer() { CORRADE_COMPARE(instanceNode->translation(), Mn::Vector3(1.f, 2.f, 3.f)); CORRADE_COMPARE(instanceNode->getSemanticId(), semanticId); + CORRADE_COMPARE(instanceNode->getBaseObjectId(), objectId); } else { // if rootNode had children originally, then stateUpdate was // applied to a sibling of the lastRootChild // get the lastRootChild before the stateUpdate @@ -351,6 +371,7 @@ void GfxReplayTest::testPlayer() { CORRADE_COMPARE(instanceNode->translation(), Mn::Vector3(1.f, 2.f, 3.f)); CORRADE_COMPARE(instanceNode->getSemanticId(), semanticId); + CORRADE_COMPARE(instanceNode->getBaseObjectId(), objectId); } } else if (keyframeIndex == 2) { // assert that no new nodes were created diff --git a/src/tests/IOTest.cpp b/src/tests/IOTest.cpp index cb10ae30d6..2336676c8b 100644 --- a/src/tests/IOTest.cpp +++ b/src/tests/IOTest.cpp @@ -437,7 +437,7 @@ void IOTest::testJsonEspTypes() { {Magnum::Vector3(1.f, 2.f, 3.f), Magnum::Quaternion::rotation(Magnum::Rad{1.f}, Magnum::Vector3(0.f, 1.f, 0.f))}, - 4}; + }; esp::io::addMember(d, "state", state, allocator); // read and compare RenderAssetInstanceState esp::gfx::replay::RenderAssetInstanceState state2;