Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework of parsing the header of a replay #143

Merged
merged 26 commits into from
May 18, 2024
Merged

Rework of parsing the header of a replay #143

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
1403423
Separate out the reading and interpreting of the file
Garanas May 6, 2024
01f0f6e
Undo lfs of fafreplay
Sheikah45 May 6, 2024
01f3bc2
Fix replays that were damaged by LFS
Garanas May 7, 2024
20155f0
Make the memory-based functions private
Garanas May 7, 2024
a5ee027
Clean up of the code
Garanas May 7, 2024
c7f75b4
Improve documentation of unknown values
Garanas May 7, 2024
bccff78
Document the header of a replay
Garanas May 7, 2024
a8e7303
First working end-to-end example of the new replay parsing
Garanas May 7, 2024
43c0572
Restructure the project, process feedback by Sheikah in #142
Garanas May 7, 2024
8558a85
Extend tests and fix a few bugs
Garanas May 7, 2024
a5467f0
Discover the byte that is set when queueing orders
Garanas May 8, 2024
d0e9566
Introduce the first semantics
Garanas May 9, 2024
dbb816b
Add interpretation of chat messages
Garanas May 9, 2024
7d4ef66
Introduce additional semantics
Garanas May 11, 2024
0f0aca2
Use atomics
Garanas May 11, 2024
cce2ee3
Rework the enums related to game options
Garanas May 11, 2024
a18552a
Remove excessive whitespace
Garanas May 11, 2024
2caafd3
Add documentation that it still requires to be implemented
Garanas May 11, 2024
07bee25
Fix typo in name
Garanas May 11, 2024
020955d
Rename 'Utils' to 'LoadUtils'
Garanas May 11, 2024
4348c64
Use a better describing exception
Garanas May 11, 2024
49ddff9
Undo formatting changes
Garanas May 11, 2024
76615a8
Remove the tokenizer of the header
Garanas May 17, 2024
706ad77
Extend pattern matching of modern Java
Garanas May 17, 2024
bdee610
DO THAT SCREAMING SNAKE CASE THING
Garanas May 17, 2024
1b9492f
Process feedback
Garanas May 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions faf-commons-data/src/main/java/com/faforever/commons/replay/ReplayContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package com.faforever.commons.replay;

import com.faforever.commons.replay.body.ReplayBody;
import com.faforever.commons.replay.header.ReplayHeader;

public record ReplayContainer(ReplayMetadata metadata, ReplayHeader header, ReplayBody body, byte[] bytes) {
}
142 changes: 68 additions & 74 deletions faf-commons-data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package com.faforever.commons.replay;

import com.faforever.commons.replay.body.event.Event;
import com.faforever.commons.replay.body.event.LuaData;
import com.faforever.commons.replay.body.event.Parser;
import com.faforever.commons.replay.body.token.Token;
import com.faforever.commons.replay.body.token.Tokenizer;
import com.faforever.commons.replay.body.ReplayBodyEvent;
import com.faforever.commons.replay.semantics.ChatMessage;
import com.faforever.commons.replay.semantics.ModeratorEvent;
import com.faforever.commons.replay.shared.LuaTable;
import com.faforever.commons.replay.body.ReplayBodyParser;
import com.faforever.commons.replay.body.ReplayBodyToken;
import com.faforever.commons.replay.body.ReplayBodyTokenizer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.BaseEncoding;
Expand Down Expand Up @@ -67,10 +69,10 @@ public class ReplayDataParser {
private List<GameOption> gameOptions;

@Getter
private List<Token> tokens;
private List<ReplayBodyToken> tokens;

@Getter
private List<Event> events;
private List<ReplayBodyEvent> events;

public ReplayDataParser(Path path, ObjectMapper objectMapper) throws IOException, CompressorException {
this.path = path;
Expand Down Expand Up @@ -151,8 +153,7 @@ private byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws
}
case ZSTD: {
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(data);
CompressorInputStream compressorInputStream = new CompressorStreamFactory()
.createCompressorInputStream(arrayInputStream);
CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream);

ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copy(compressorInputStream, out);
Expand All @@ -167,22 +168,18 @@ private byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws
@SuppressWarnings("unchecked")
private void parseHeader(LittleEndianDataInputStream dataStream) throws IOException {
replayPatchFieldId = readString(dataStream);
dataStream.skipBytes(3);
String arg13 = readString(dataStream); // always \r\n

String[] split = readString(dataStream).split("\\r\\n");
String replayVersionId = split[0];
map = split[1];
dataStream.skipBytes(4);
String arg23 = readString((dataStream)); // always \r\n and some unknown character

int numberOfMods = dataStream.readInt();
int sizeModsInBytes = dataStream.readInt();
mods = (Map<String, Map<String, ?>>) parseLua(dataStream);

int scenarioSize = dataStream.readInt();
this.gameOptions = ((Map<String, Object>) parseLua(dataStream)).entrySet().stream()
.filter(entry -> "Options".equals(entry.getKey()))
.flatMap(entry -> ((Map<String, Object>) entry.getValue()).entrySet().stream())
.map(entry -> new GameOption(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());
int sizeGameOptionsInBytes = dataStream.readInt();
this.gameOptions = ((Map<String, Object>) parseLua(dataStream)).entrySet().stream().filter(entry -> "Options".equals(entry.getKey())).flatMap(entry -> ((Map<String, Object>) entry.getValue()).entrySet().stream()).map(entry -> new GameOption(entry.getKey(), entry.getValue())).collect(Collectors.toList());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

leave this wrapped as it is more readable

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's the formatter!! 😛

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Processed in:


int numberOfSources = dataStream.readUnsignedByte();

Expand All @@ -197,7 +194,7 @@ private void parseHeader(LittleEndianDataInputStream dataStream) throws IOExcept

int numberOfArmies = dataStream.readUnsignedByte();
for (int i = 0; i < numberOfArmies; i++) {
dataStream.skipBytes(4);
int sizePlayerDataInBytes = dataStream.readInt();
Map<String, Object> playerData = (Map<String, Object>) parseLua(dataStream);
int playerSource = dataStream.readUnsignedByte();

Expand All @@ -212,38 +209,38 @@ private void parseHeader(LittleEndianDataInputStream dataStream) throws IOExcept
randomSeed = dataStream.readInt();
}

private void interpretEvents(List<Event> events) {
private void interpretEvents(List<ReplayBodyEvent> events) {
Integer player = -1;
boolean desync = false;
String previousChecksum = null;
int previousTick = -1;

Map<Integer, Integer> lastTicks = new HashMap<>();

for (Event event : events) {
for (ReplayBodyEvent event : events) {

switch (event) {
case Event.Unprocessed(Token token, String reason) -> {
case ReplayBodyEvent.Unprocessed(ReplayBodyToken token, String reason) -> {

}

case Event.ProcessingError(Token token, Exception exception) -> {
case ReplayBodyEvent.ProcessingError(ReplayBodyToken token, Exception exception) -> {

}

case Event.Advance(int ticksToAdvance) -> {
case ReplayBodyEvent.Advance(int ticksToAdvance) -> {
ticks += ticksToAdvance;
}

case Event.SetCommandSource(int playerIndex) -> {
case ReplayBodyEvent.SetCommandSource(int playerIndex) -> {
player = playerIndex;
}

case Event.CommandSourceTerminated() -> {
case ReplayBodyEvent.CommandSourceTerminated() -> {
lastTicks.put(player, ticks);
}

case Event.VerifyChecksum(String hash, int tick) -> {
case ReplayBodyEvent.VerifyChecksum(String hash, int tick) -> {
desync = tick == previousTick && !Objects.equals(previousChecksum, hash);
previousChecksum = hash;
previousTick = ticks;
Expand All @@ -254,117 +251,115 @@ private void interpretEvents(List<Event> events) {
}
}

case Event.RequestPause() -> {
case ReplayBodyEvent.RequestPause() -> {

}

case Event.RequestResume() -> {
case ReplayBodyEvent.RequestResume() -> {

}

case Event.SingleStep() -> {
case ReplayBodyEvent.SingleStep() -> {

}

case Event.CreateUnit(int playerIndex, String blueprintId, float px, float pz, float heading) -> {
case ReplayBodyEvent.CreateUnit(int playerIndex, String blueprintId, float px, float pz, float heading) -> {

}

case Event.CreateProp(String blueprintId, float px, float pz, float heading) -> {
case ReplayBodyEvent.CreateProp(String blueprintId, float px, float pz, float heading) -> {

}

case Event.DestroyEntity(int entityId) -> {
case ReplayBodyEvent.DestroyEntity(int entityId) -> {

}

case Event.WarpEntity(int entityId, float px, float py, float pz) -> {
case ReplayBodyEvent.WarpEntity(int entityId, float px, float py, float pz) -> {

}

case Event.ProcessInfoPair(int entityId, String arg1, String arg2) -> {
case ReplayBodyEvent.ProcessInfoPair(int entityId, String arg1, String arg2) -> {

}

case Event.IssueCommand(Event.CommandUnits commandUnits, Event.CommandData commandData) -> {
commandsPerMinuteByPlayer
.computeIfAbsent(player, p -> new HashMap<>())
.computeIfAbsent(ticks, t -> new AtomicInteger())
.incrementAndGet();
case ReplayBodyEvent.IssueCommand(
ReplayBodyEvent.CommandUnits commandUnits, ReplayBodyEvent.CommandData commandData
) -> {
commandsPerMinuteByPlayer.computeIfAbsent(player, p -> new HashMap<>()).computeIfAbsent(ticks, t -> new AtomicInteger()).incrementAndGet();
}

case Event.IssueFactoryCommand(
Event.CommandUnits commandUnits, Event.CommandData commandData
case ReplayBodyEvent.IssueFactoryCommand(
ReplayBodyEvent.CommandUnits commandUnits, ReplayBodyEvent.CommandData commandData
) -> {
commandsPerMinuteByPlayer
.computeIfAbsent(player, p -> new HashMap<>())
.computeIfAbsent(ticks, t -> new AtomicInteger())
.incrementAndGet();
commandsPerMinuteByPlayer.computeIfAbsent(player, p -> new HashMap<>()).computeIfAbsent(ticks, t -> new AtomicInteger()).incrementAndGet();
}

case Event.IncreaseCommandCount(int commandId, int delta) -> {
case ReplayBodyEvent.IncreaseCommandCount(int commandId, int delta) -> {

}

case Event.DecreaseCommandCount(int commandId, int delta) -> {
case ReplayBodyEvent.DecreaseCommandCount(int commandId, int delta) -> {

}

case Event.SetCommandTarget(int commandId, Event.CommandTarget commandTarget) -> {
case ReplayBodyEvent.SetCommandTarget(int commandId, ReplayBodyEvent.CommandTarget commandTarget) -> {

}

case Event.SetCommandType(int commandId, int targetId) -> {
case ReplayBodyEvent.SetCommandType(int commandId, int targetId) -> {

}

case Event.SetCommandCells(int commandId, Object parametersLua, float px, float py, float pz) -> {
case ReplayBodyEvent.SetCommandCells(int commandId, Object parametersLua, float px, float py, float pz) -> {

}

case Event.RemoveCommandFromQueue(int commandId, int unitId) -> {
case ReplayBodyEvent.RemoveCommandFromQueue(int commandId, int unitId) -> {

}

case Event.DebugCommand() -> {
case ReplayBodyEvent.DebugCommand(
String command, float px, float py, float pz, byte focusArmy, ReplayBodyEvent.CommandUnits units
) -> {

}

case Event.ExecuteLuaInSim(String luaCode) -> {
case ReplayBodyEvent.ExecuteLuaInSim(String luaCode) -> {

}

case Event.LuaSimCallback(
String func, LuaData.Table parametersLua, Event.CommandUnits commandUnits
case ReplayBodyEvent.LuaSimCallback(
String func, LuaTable.Table parametersLua, ReplayBodyEvent.CommandUnits commandUnits
) when func.equals("GiveResourcesToPlayer") -> {
parseGiveResourcesToPlayer(parametersLua);
}

case Event.LuaSimCallback(
String func, LuaData.Table parametersLua, Event.CommandUnits commandUnits
case ReplayBodyEvent.LuaSimCallback(
String func, LuaTable.Table parametersLua, ReplayBodyEvent.CommandUnits commandUnits
) when func.equals("ModeratorEvent") -> {
parseModeratorEvent(parametersLua, player);
}

case Event.LuaSimCallback(
String func, LuaData parametersLua, Event.CommandUnits commandUnits
case ReplayBodyEvent.LuaSimCallback(
String func, LuaTable parametersLua, ReplayBodyEvent.CommandUnits commandUnits
) -> {

}

case Event.EndGame() -> {
case ReplayBodyEvent.EndGame() -> {

}

}
}
}

private void parseGiveResourcesToPlayer(LuaData.Table lua) {
private void parseGiveResourcesToPlayer(LuaTable.Table lua) {
if (lua.value().containsKey("Msg") && lua.value().containsKey("From") && lua.value().containsKey("Sender")) {

// TODO: use the command source (player value) instead of the values from the callback. The values from the callback can be manipulated
if (!(lua.value().get("From") instanceof LuaData.Number(float luaFromArmy))) {
if (!(lua.value().get("From") instanceof LuaTable.Number(float luaFromArmy))) {
return;
}

Expand All @@ -373,20 +368,20 @@ private void parseGiveResourcesToPlayer(LuaData.Table lua) {
return;
}

if (!(lua.value().get("Msg") instanceof LuaData.Table(Map<String, LuaData> luaMsg))) {
if (!(lua.value().get("Msg") instanceof LuaTable.Table(Map<String, LuaTable> luaMsg))) {
return;
}

if (!(lua.value().get("Sender") instanceof LuaData.String(String luaSender))) {
if (!(lua.value().get("Sender") instanceof LuaTable.String(String luaSender))) {
return;
}

// This can either be a player name or a Map of something, in which case it's actually giving resources
if (!(luaMsg.get("to") instanceof LuaData.String(String luaMsgReceiver))) {
if (!(luaMsg.get("to") instanceof LuaTable.String(String luaMsgReceiver))) {
return;
}

if (!(luaMsg.get("text") instanceof LuaData.String(String luaMsgText))) {
if (!(luaMsg.get("text") instanceof LuaTable.String(String luaMsgText))) {
return;
}

Expand All @@ -398,18 +393,18 @@ private void parseGiveResourcesToPlayer(LuaData.Table lua) {
}


void parseModeratorEvent(LuaData.Table lua, Integer player) {
void parseModeratorEvent(LuaTable.Table lua, Integer player) {
String messageContent = null;
String playerNameFromArmy = null;
String playerNameFromCommandSource = null;
Integer activeCommandSource = null;
Integer fromArmy = null;

if (lua.value().get("Message") instanceof LuaData.String(String luaMessage)) {
if (lua.value().get("Message") instanceof LuaTable.String(String luaMessage)) {
messageContent = luaMessage;
}

if (lua.value().get("From") instanceof LuaData.Number(float luaFrom)) {
if (lua.value().get("From") instanceof LuaTable.Number(float luaFrom)) {
fromArmy = (int) luaFrom - 1;


Expand All @@ -431,8 +426,7 @@ void parseModeratorEvent(LuaData.Table lua, Integer player) {
}
}

moderatorEvents.add(new ModeratorEvent(tickToTime(ticks), activeCommandSource, fromArmy,
messageContent, playerNameFromArmy, playerNameFromCommandSource));
moderatorEvents.add(new ModeratorEvent(tickToTime(ticks), activeCommandSource, fromArmy, messageContent, playerNameFromArmy, playerNameFromCommandSource));
}

private Duration tickToTime(int tick) {
Expand All @@ -443,9 +437,9 @@ private void parse() throws IOException, CompressorException {
readReplayData(path);
try (LittleEndianDataInputStream dataStream = new LittleEndianDataInputStream(new ByteArrayInputStream(data))) {
parseHeader(dataStream);
tokens = Tokenizer.tokenize(dataStream);
tokens = ReplayBodyTokenizer.tokenize(dataStream);
}
events = Parser.parseTokens(tokens);
events = ReplayBodyParser.parseTokens(tokens);
interpretEvents(events);
}
}
Loading