Rework of parsing the header of a replay

faf-commons-data/src/main/java/com/faforever/commons/replay/RegisteredEvent.java

@@ -0,0 +1,15 @@
+package com.faforever.commons.replay;
+
+import com.faforever.commons.replay.body.Event;
+import com.faforever.commons.replay.header.Source;
+
+/**
+ * Combines the tick and source of an event. The tick represents when the event was registered. The source represents who authorised the event.
+ *
+ * @param tick
+ * @param source
+ * @param event
+ * @see Event
+ */
+public record RegisteredEvent(int tick, Source source, Event event) {
+}

faf-commons-data/src/main/java/com/faforever/commons/replay/ReplayContainer.java

@@ -0,0 +1,15 @@
+package com.faforever.commons.replay;
+
+import com.faforever.commons.replay.header.ReplayHeader;
+
+import java.util.List;
+
+/**
+ * A container of all the information that a replay may hold once parsed.
+ *
+ * @param metadata
+ * @param header
+ * @param registeredEvents
+ */
+public record ReplayContainer(ReplayMetadata metadata, ReplayHeader header, List<RegisteredEvent> registeredEvents) {
+}

faf-commons-data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java

@@ -1,10 +1,10 @@
 package com.faforever.commons.replay;
 
-import com.faforever.commons.replay.body.event.Event;
-import com.faforever.commons.replay.body.event.LuaData;
-import com.faforever.commons.replay.body.event.Parser;
-import com.faforever.commons.replay.body.token.Token;
-import com.faforever.commons.replay.body.token.Tokenizer;
+import com.faforever.commons.replay.body.Event;
+import com.faforever.commons.replay.shared.LuaData;
+import com.faforever.commons.replay.body.ReplayBodyParser;
+import com.faforever.commons.replay.body.ReplayBodyToken;
+import com.faforever.commons.replay.body.ReplayBodyTokenizer;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.BaseEncoding;
@@ -67,7 +67,7 @@ public class ReplayDataParser {
   private List<GameOption> gameOptions;
 
   @Getter
-  private List<Token> tokens;
+  private List<ReplayBodyToken> tokens;
 
   @Getter
   private List<Event> events;
@@ -151,8 +151,7 @@ private byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws
       }
       case ZSTD: {
         ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(data);
-        CompressorInputStream compressorInputStream = new CompressorStreamFactory()
-          .createCompressorInputStream(arrayInputStream);
+        CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream);
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         IOUtils.copy(compressorInputStream, out);
@@ -167,17 +166,17 @@ private byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws
   @SuppressWarnings("unchecked")
   private void parseHeader(LittleEndianDataInputStream dataStream) throws IOException {
     replayPatchFieldId = readString(dataStream);
-    dataStream.skipBytes(3);
+    String arg13 = readString(dataStream); // always \r\n
 
     String[] split = readString(dataStream).split("\\r\\n");
     String replayVersionId = split[0];
     map = split[1];
-    dataStream.skipBytes(4);
+    String arg23 = readString((dataStream)); // always \r\n and some unknown character
 
-    int numberOfMods = dataStream.readInt();
+    int sizeModsInBytes = dataStream.readInt();
     mods = (Map<String, Map<String, ?>>) parseLua(dataStream);
 
-    int scenarioSize = dataStream.readInt();
+    int sizeGameOptionsInBytes = dataStream.readInt();
     this.gameOptions = ((Map<String, Object>) parseLua(dataStream)).entrySet().stream()
       .filter(entry -> "Options".equals(entry.getKey()))
       .flatMap(entry -> ((Map<String, Object>) entry.getValue()).entrySet().stream())
@@ -197,7 +196,7 @@ private void parseHeader(LittleEndianDataInputStream dataStream) throws IOExcept
 
     int numberOfArmies = dataStream.readUnsignedByte();
     for (int i = 0; i < numberOfArmies; i++) {
-      dataStream.skipBytes(4);
+      int sizePlayerDataInBytes = dataStream.readInt();
       Map<String, Object> playerData = (Map<String, Object>) parseLua(dataStream);
       int playerSource = dataStream.readUnsignedByte();
 
@@ -223,11 +222,11 @@ private void interpretEvents(List<Event> events) {
     for (Event event : events) {
 
       switch (event) {
-        case Event.Unprocessed(Token token, String reason) -> {
+        case Event.Unprocessed(ReplayBodyToken token, String reason) -> {
 
         }
 
-        case Event.ProcessingError(Token token, Exception exception) -> {
+        case Event.ProcessingError(ReplayBodyToken token, Exception exception) -> {
 
         }
 
@@ -286,7 +285,9 @@ private void interpretEvents(List<Event> events) {
 
         }
 
-        case Event.IssueCommand(Event.CommandUnits commandUnits, Event.CommandData commandData) -> {
+        case Event.IssueCommand(
+          Event.CommandUnits commandUnits, Event.CommandData commandData
+        ) -> {
           commandsPerMinuteByPlayer
             .computeIfAbsent(player, p -> new HashMap<>())
             .computeIfAbsent(ticks, t -> new AtomicInteger())
@@ -326,7 +327,9 @@ private void interpretEvents(List<Event> events) {
 
         }
 
-        case Event.DebugCommand() -> {
+        case Event.DebugCommand(
+          String command, float px, float py, float pz, byte focusArmy, Event.CommandUnits units
+        ) -> {
 
         }
 
@@ -431,8 +434,7 @@ void parseModeratorEvent(LuaData.Table lua, Integer player) {
       }
     }
 
-    moderatorEvents.add(new ModeratorEvent(tickToTime(ticks), activeCommandSource, fromArmy,
-      messageContent, playerNameFromArmy, playerNameFromCommandSource));
+    moderatorEvents.add(new ModeratorEvent(tickToTime(ticks), activeCommandSource, fromArmy, messageContent, playerNameFromArmy, playerNameFromCommandSource));
   }
 
   private Duration tickToTime(int tick) {
@@ -443,9 +445,9 @@ private void parse() throws IOException, CompressorException {
     readReplayData(path);
     try (LittleEndianDataInputStream dataStream = new LittleEndianDataInputStream(new ByteArrayInputStream(data))) {
       parseHeader(dataStream);
-      tokens = Tokenizer.tokenize(dataStream);
+      tokens = ReplayBodyTokenizer.tokenize(dataStream);
     }
-    events = Parser.parseTokens(tokens);
+    events = ReplayBodyParser.parseTokens(tokens);
     interpretEvents(events);
   }
 }

faf-commons-data/src/main/java/com/faforever/commons/replay/ReplayLoader.java

@@ -0,0 +1,123 @@
+package com.faforever.commons.replay;
+
+import com.faforever.commons.replay.body.Event;
+import com.faforever.commons.replay.body.ReplayBodyParser;
+import com.faforever.commons.replay.body.ReplayBodyToken;
+import com.faforever.commons.replay.body.ReplayBodyTokenizer;
+import com.faforever.commons.replay.header.*;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.google.common.io.BaseEncoding;
+import com.google.common.io.LittleEndianDataInputStream;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.commons.compress.utils.IOUtils;
+import org.jetbrains.annotations.Contract;
+import org.jetbrains.annotations.NotNull;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class ReplayLoader {
+
+  @Contract(pure = true)
+  private static ReplayHeader loadSCFAReplayHeader(LittleEndianDataInputStream stream) throws IOException {
+    return ReplayHeaderParser.parse(stream);
+  }
+
+  @Contract(pure = true)
+  private static @NotNull List<RegisteredEvent> loadSCFAReplayBody(List<Source> sources, LittleEndianDataInputStream stream) throws IOException {
+    List<ReplayBodyToken> bodyTokens = ReplayBodyTokenizer.tokenize(stream);
+    List<Event> bodyEvents = ReplayBodyParser.parseTokens(bodyTokens);
+    return ReplaySemantics.registerEvents(sources, bodyEvents);
+  }
+
+  @Contract(pure = true)
+  private static ReplayContainer loadSCFAReplayFromMemory(ReplayMetadata metadata, byte[] scfaReplayBytes) throws IOException {
+    try (LittleEndianDataInputStream stream = new LittleEndianDataInputStream((new ByteArrayInputStream(scfaReplayBytes)))) {
+      ReplayHeader replayHeader = loadSCFAReplayHeader(stream);
+      List<RegisteredEvent> replayBody = loadSCFAReplayBody(replayHeader.sources(), stream);
+
+      if (stream.available() > 0) {
+        throw new EOFException();
+      }
+
+      return new ReplayContainer(metadata, replayHeader, replayBody);
+    }
+  }
+
+  public static ReplayContainer loadSCFAReplayFromDisk(Path scfaReplayFile) throws IOException, IllegalArgumentException {
+    if (!scfaReplayFile.toString().toLowerCase().endsWith("scfareplay")) {
+      throw new IllegalArgumentException("Unknown file format: " + scfaReplayFile.getFileName());
+    }
+
+    byte[] bytes = Files.readAllBytes(scfaReplayFile);
+    return loadSCFAReplayFromMemory(null, bytes);
+  }
+
+  @Contract(pure = true)
+  private static ReplayContainer loadFAFReplayFromMemory(byte[] fafReplayBytes) throws IOException, CompressorException {
+    int separator = findSeparatorIndex(fafReplayBytes);
+    byte[] metadataBytes = Arrays.copyOfRange(fafReplayBytes, 0, separator);
+    String metadataString = new String(metadataBytes, StandardCharsets.UTF_8);
+
+    ObjectMapper parsedMetadata = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    ReplayMetadata replayMetadata = parsedMetadata.readValue(metadataString, ReplayMetadata.class);
+
+    byte[] compressedReplayBytes = Arrays.copyOfRange(fafReplayBytes, separator + 1, fafReplayBytes.length);
+    byte[] scfaReplayBytes = decompress(compressedReplayBytes, replayMetadata);
+
+    return loadSCFAReplayFromMemory(replayMetadata, scfaReplayBytes);
+  }
+
+  public static ReplayContainer loadFAFReplayFromDisk(Path fafReplayFile) throws IOException, CompressorException, IllegalArgumentException {
+    if (!fafReplayFile.toString().toLowerCase().endsWith("fafreplay")) {
+      throw new IllegalArgumentException("Unknown file format: " + fafReplayFile.getFileName());
+    }
+
+    byte[] fafReplayBytes = Files.readAllBytes(fafReplayFile);
+    return loadFAFReplayFromMemory(fafReplayBytes);
+  }
+
+  private static int findSeparatorIndex(byte[] replayData) {
+    int headerEnd;
+    for (headerEnd = 0; headerEnd < replayData.length; headerEnd++) {
+      if (replayData[headerEnd] == '\n') {
+        return headerEnd;
+      }
+    }
+    throw new IllegalArgumentException("Missing separator between replay header and body");
+  }
+
+  private static byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws IOException, CompressorException {
+    CompressionType compressionType = Objects.requireNonNullElse(metadata.getCompression(), CompressionType.QTCOMPRESS);
+
+    switch (compressionType) {
+      case QTCOMPRESS: {
+        return QtCompress.qUncompress(BaseEncoding.base64().decode(new String(data)));
+      }
+      case ZSTD: {
+        ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(data);
+        CompressorInputStream compressorInputStream = new CompressorStreamFactory()
+          .createCompressorInputStream(arrayInputStream);
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copy(compressorInputStream, out);
+        return out.toByteArray();
+      }
+      case UNKNOWN:
+      default:
+        throw new IOException("Unknown replay format in replay file");
+    }
+  }
+}