Skip to content

Commit

Permalink
Add instance metadata to gfx-replay. (#2377)
Browse files Browse the repository at this point in the history 
* Add instance metadata to gfx-replay.

* Update metadata like state updates rather than creations.

* Fix json parsing.
  • Loading branch information
@0mdc
0mdc committed May 2, 2024
1 parent 2ac5b9f commit a37dc49
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 36 deletions.
17 changes: 14 additions & 3 deletions src/esp/gfx/replay/Keyframe.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,21 @@ struct Transform {
*/
struct RenderAssetInstanceState {
Transform absTransform; // localToWorld
// note we currently only support semanticId per instance, not per drawable
int semanticId = ID_UNDEFINED;
// note we don't currently support runtime changes to lightSetupKey
bool operator==(const RenderAssetInstanceState& rhs) const {
return absTransform == rhs.absTransform && semanticId == rhs.semanticId;
return absTransform == rhs.absTransform;
}
};

/**
* @brief Metadata associated with an instance.
*/
struct InstanceMetadata {
int objectId = ID_UNDEFINED;
int semanticId = ID_UNDEFINED;

bool operator==(const InstanceMetadata& rhs) const {
return objectId == rhs.objectId && semanticId == rhs.semanticId;
}
};

Expand Down Expand Up @@ -67,6 +77,7 @@ struct Keyframe {
esp::assets::RenderAssetInstanceCreationInfo>>
creations;
std::vector<RenderAssetInstanceKey> deletions;
std::vector<std::pair<RenderAssetInstanceKey, InstanceMetadata>> metadata;
std::vector<std::pair<RenderAssetInstanceKey, RenderAssetInstanceState>>
stateUpdates;
std::vector<RigUpdate> rigUpdates;
Expand Down
39 changes: 28 additions & 11 deletions src/esp/gfx/replay/Player.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ std::string removeMaterialOverrideFromFilepathAndWarn(const std::string& src) {

static_assert(std::is_nothrow_move_constructible<Player>::value, "");

void AbstractPlayerImplementation::setNodeSemanticId(NodeHandle, unsigned) {}
void AbstractPlayerImplementation::setNodeMetadata(NodeHandle,
const InstanceMetadata&) {}

void AbstractPlayerImplementation::changeLightSetup(const LightSetup&) {}

Expand Down Expand Up @@ -75,10 +76,15 @@ Mn::Matrix4 AbstractSceneGraphPlayerImplementation::hackGetNodeTransform(
return (*reinterpret_cast<scene::SceneNode*>(node)).transformation();
}

void AbstractSceneGraphPlayerImplementation::setNodeSemanticId(
void AbstractSceneGraphPlayerImplementation::setNodeMetadata(
const NodeHandle node,
const unsigned id) {
setSemanticIdForSubtree(reinterpret_cast<scene::SceneNode*>(node), id);
const InstanceMetadata& metadata) {
setSemanticInfoForSubtree(reinterpret_cast<scene::SceneNode*>(node),
{
metadata.semanticId,
metadata.objectId,
ID_UNDEFINED,
});
}

void AbstractPlayerImplementation::createRigInstance(
Expand Down Expand Up @@ -186,7 +192,7 @@ void Player::clearFrame() {
implementation_->deleteAssetInstances(createdInstances_);
createdInstances_.clear();
assetInfos_.clear();
creationInfos_.clear();
creationRecords_.clear();
frameIndex_ = -1;
}

Expand Down Expand Up @@ -235,7 +241,16 @@ void Player::applyKeyframe(const Keyframe& keyframe) {
const auto& instanceKey = pair.first;
CORRADE_INTERNAL_ASSERT(createdInstances_.count(instanceKey) == 0);
createdInstances_[instanceKey] = node;
creationInfos_[instanceKey] = adjustedCreation;

creationRecords_[instanceKey] =
CreationRecord{adjustedCreation, InstanceMetadata()};
}

for (const auto& pair : keyframe.metadata) {
const auto& instanceKey = pair.first;
implementation_->setNodeMetadata(createdInstances_[instanceKey],
pair.second);
creationRecords_[instanceKey].metadata = pair.second;
}

hackProcessDeletions(keyframe);
Expand All @@ -250,7 +265,6 @@ void Player::applyKeyframe(const Keyframe& keyframe) {
const auto& state = pair.second;
implementation_->setNodeTransform(node, state.absTransform.translation,
state.absTransform.rotation);
implementation_->setNodeSemanticId(node, state.semanticId);
}

for (const auto& rigUpdate : keyframe.rigUpdates) {
Expand Down Expand Up @@ -282,11 +296,11 @@ void Player::hackProcessDeletions(const Keyframe& keyframe) {
implementation_->deleteAssetInstance(it->second);
createdInstances_.erase(deletionInstanceKey);

int rigId = creationInfos_[deletionInstanceKey].rigId;
int rigId = creationRecords_[deletionInstanceKey].creationInfo.rigId;
if (rigId != ID_UNDEFINED) {
implementation_->deleteRigInstance(rigId);
}
creationInfos_.erase(deletionInstanceKey);
creationRecords_.erase(deletionInstanceKey);
}
} else if (keyframe.deletions.size() > 0) {
// Cache latest transforms
Expand All @@ -308,14 +322,17 @@ void Player::hackProcessDeletions(const Keyframe& keyframe) {
continue;
}
createdInstances_.erase(createInstanceIt);
creationInfos_.erase(deletion);
creationRecords_.erase(deletion);
}

for (const auto& pair : createdInstances_) {
const auto key = pair.first;
const auto& creationInfo = creationInfos_[key];
const auto& creationRecord = creationRecords_[key];
const auto& creationInfo = creationRecord.creationInfo;
auto* instance = implementation_->loadAndCreateRenderAssetInstance(
assetInfos_[creationInfo.filepath], creationInfo);
const auto& metadata = creationRecord.metadata;
implementation_->setNodeMetadata(instance, metadata);

// Replace dangling reference
createdInstances_[key] = instance;
Expand Down
17 changes: 11 additions & 6 deletions src/esp/gfx/replay/Player.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ namespace replay {

class Player;

struct CreationRecord {
assets::RenderAssetInstanceCreationInfo creationInfo;
InstanceMetadata metadata;
};

/**
@brief Node handle

Expand Down Expand Up @@ -112,13 +117,14 @@ class AbstractPlayerImplementation {
virtual Mn::Matrix4 hackGetNodeTransform(NodeHandle node) const = 0;

/**
* @brief Set node semantic ID
* @brief Set node metadata.
*
* The @p handle is expected to be returned from an earlier call to
* @ref loadAndCreateRenderAssetInstance() on the same instance. Default
* implementation does nothing.
*/
virtual void setNodeSemanticId(NodeHandle node, unsigned id);
virtual void setNodeMetadata(NodeHandle node,
const InstanceMetadata& metadata);

/**
* @brief Change light setup
Expand Down Expand Up @@ -176,7 +182,8 @@ class AbstractSceneGraphPlayerImplementation

Mn::Matrix4 hackGetNodeTransform(NodeHandle node) const override;

void setNodeSemanticId(NodeHandle node, unsigned id) override;
void setNodeMetadata(NodeHandle node,
const InstanceMetadata& metadata) override;
};

/**
Expand Down Expand Up @@ -304,9 +311,7 @@ class Player {
std::vector<Keyframe> keyframes_;
std::unordered_map<std::string, esp::assets::AssetInfo> assetInfos_;
std::unordered_map<RenderAssetInstanceKey, NodeHandle> createdInstances_;
std::unordered_map<RenderAssetInstanceKey,
assets::RenderAssetInstanceCreationInfo>
creationInfos_;
std::unordered_map<RenderAssetInstanceKey, CreationRecord> creationRecords_;
std::unordered_map<RenderAssetInstanceKey, Mn::Matrix4> latestTransformCache_;
std::set<std::string> failedFilepaths_;

Expand Down
24 changes: 18 additions & 6 deletions src/esp/gfx/replay/Recorder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ void Recorder::onCreateRenderAssetInstance(
// manually later if necessary.
NodeDeletionHelper* deletionHelper = new NodeDeletionHelper{*node, this};

instanceRecords_.emplace_back(InstanceRecord{node, instanceKey,
Corrade::Containers::NullOpt,
deletionHelper, creation.rigId});
instanceRecords_.emplace_back(InstanceRecord{
node, instanceKey, Corrade::Containers::NullOpt,
Corrade::Containers::NullOpt, deletionHelper, creation.rigId});
}

void Recorder::onCreateRigInstance(int rigId, const Rig& rig) {
Expand Down Expand Up @@ -214,7 +214,13 @@ RenderAssetInstanceState Recorder::getInstanceState(
const scene::SceneNode* node) {
const auto absTransformMat = node->absoluteTransformation();
const auto transform = ::createReplayTransform(absTransformMat);
return RenderAssetInstanceState{transform, node->getSemanticId()};
return RenderAssetInstanceState{transform};
}

InstanceMetadata Recorder::getInstanceMetadata(const scene::SceneNode* node) {
const auto objectId = node->getBaseObjectId();
const auto semanticId = node->getSemanticId();
return InstanceMetadata{objectId, semanticId};
}

void Recorder::updateStates() {
Expand All @@ -230,6 +236,11 @@ void Recorder::updateInstanceStates() {
state);
instanceRecord.recentState = state;
}
auto metadata = getInstanceMetadata(instanceRecord.node);
if (!instanceRecord.metadata || metadata != instanceRecord.metadata) {
getKeyframe().metadata.emplace_back(instanceRecord.instanceKey, metadata);
instanceRecord.metadata = metadata;
}
}
}

Expand Down Expand Up @@ -332,10 +343,11 @@ void Recorder::consolidateSavedKeyframes() {
// consolidate saved keyframes into current keyframe
addLoadsCreationsDeletions(savedKeyframes_.begin(), savedKeyframes_.end(),
&getKeyframe());
// clear instanceRecord.recentState to ensure updates get included in the next
// saved keyframe.
// clear instanceRecord.recentState and metadata to ensure updates get
// included in the next saved keyframe.
for (auto& instanceRecord : instanceRecords_) {
instanceRecord.recentState = Corrade::Containers::NullOpt;
instanceRecord.metadata = Corrade::Containers::NullOpt;
}
savedKeyframes_.clear();
}
Expand Down
2 changes: 2 additions & 0 deletions src/esp/gfx/replay/Recorder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ class Recorder {
scene::SceneNode* node = nullptr;
RenderAssetInstanceKey instanceKey = ID_UNDEFINED;
Corrade::Containers::Optional<RenderAssetInstanceState> recentState;
Corrade::Containers::Optional<InstanceMetadata> metadata;
NodeDeletionHelper* deletionHelper = nullptr;
int rigId = ID_UNDEFINED;
};
Expand All @@ -198,6 +199,7 @@ class Recorder {
RenderAssetInstanceKey getNewInstanceKey();
int findInstance(const scene::SceneNode* queryNode);
RenderAssetInstanceState getInstanceState(const scene::SceneNode* node);
InstanceMetadata getInstanceMetadata(const scene::SceneNode* node);
void updateStates();
void updateInstanceStates();
void updateRigInstanceStates();
Expand Down
25 changes: 25 additions & 0 deletions src/esp/io/JsonEspTypes.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,17 @@ JsonGenericValue toJsonValue(const gfx::replay::Keyframe& keyframe,
io::addMember(obj, "stateUpdates", stateUpdatesArray, allocator);
}

if (!keyframe.metadata.empty()) {
JsonGenericValue metadataArray(rapidjson::kArrayType);
for (const auto& pair : keyframe.metadata) {
JsonGenericValue metadataObj(rapidjson::kObjectType);
io::addMember(metadataObj, "instanceKey", pair.first, allocator);
io::addMember(metadataObj, "metadata", pair.second, allocator);
metadataArray.PushBack(metadataObj, allocator);
}
io::addMember(obj, "metadata", metadataArray, allocator);
}

if (!keyframe.rigUpdates.empty()) {
JsonGenericValue rigUpdatesArray(rapidjson::kArrayType);
for (const auto& rig : keyframe.rigUpdates) {
Expand Down Expand Up @@ -132,6 +143,20 @@ bool fromJsonValue(const JsonGenericValue& obj,
}
}

itr = obj.FindMember("metadata");
if (itr != obj.MemberEnd()) {
const JsonGenericValue& metadataArray = itr->value;
keyframe.metadata.reserve(metadataArray.Size());
for (const auto& metadataObj : metadataArray.GetArray()) {
std::pair<gfx::replay::RenderAssetInstanceKey,
gfx::replay::InstanceMetadata>
pair;
io::readMember(metadataObj, "instanceKey", pair.first);
io::readMember(metadataObj, "metadata", pair.second);
keyframe.metadata.emplace_back(std::move(pair));
}
}

itr = obj.FindMember("rigUpdates");
if (itr != obj.MemberEnd()) {
const JsonGenericValue& rigUpdatesArray = itr->value;
Expand Down
18 changes: 16 additions & 2 deletions src/esp/io/JsonEspTypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,22 @@ inline bool fromJsonValue(const JsonGenericValue& obj,
return true;
}

inline JsonGenericValue toJsonValue(const esp::gfx::replay::InstanceMetadata& x,
JsonAllocator& allocator) {
JsonGenericValue obj(rapidjson::kObjectType);
addMember(obj, "objectId", x.objectId, allocator);
addMember(obj, "semanticId", x.semanticId, allocator);
return obj;
}

inline bool fromJsonValue(const JsonGenericValue& obj,
esp::gfx::replay::InstanceMetadata& x) {
bool success = true;
success &= readMember(obj, "objectId", x.objectId);
success &= readMember(obj, "semanticId", x.semanticId);
return success;
}

inline JsonGenericValue toJsonValue(const esp::gfx::replay::Transform& x,
JsonAllocator& allocator) {
JsonGenericValue obj(rapidjson::kObjectType);
Expand All @@ -161,14 +177,12 @@ inline JsonGenericValue toJsonValue(
JsonAllocator& allocator) {
JsonGenericValue obj(rapidjson::kObjectType);
addMember(obj, "absTransform", x.absTransform, allocator);
addMember(obj, "semanticId", x.semanticId, allocator);
return obj;
}

inline bool fromJsonValue(const JsonGenericValue& obj,
esp::gfx::replay::RenderAssetInstanceState& x) {
readMember(obj, "absTransform", x.absTransform);
readMember(obj, "semanticId", x.semanticId);
return true;
}

Expand Down
Loading

0 comments on commit a37dc49

Please sign in to comment.