Implement collision_resolver for the flat_path_table

CollisionResolver.bt

@@ -0,0 +1,26 @@
+LittleEndian();
+
+typedef struct {
+    local int64 pos = FTell();
+    uint inode;
+    uint type;
+    uint namelen;
+    uint entsize;
+    char name[namelen];
+    char pad[entsize - (FTell() - pos)];
+} Entry;
+
+while (!FEof())
+{
+    Entry e;
+    if(ReadInt() == 0)
+    {
+        while(!FEof()){
+            FSeek(FTell() + 4);
+            if(ReadInt() != 0)
+            {
+                break;
+            }
+        }
+    }
+}

LibOrbisPkg/PFS/FlatPathTable.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.IO;
+using System.Text;
 using LibOrbisPkg.Util;
 
 namespace LibOrbisPkg.PFS
@@ -9,8 +11,69 @@ namespace LibOrbisPkg.PFS
   /// Represents the flat_path_table file, which is a mapping of filename hash to inode number
   /// that the Orbis OS can use to speed up lookups.
   /// </summary>
-  class FlatPathTable
+  public class FlatPathTable
   {
+    public static bool HasCollision(List<FSNode> nodes)
+    {
+      var hashSet = new HashSet<uint>();
+      foreach(var n in nodes)
+      {
+        var hash = HashFunction(n.FullPath());
+        if (hashSet.Contains(hash))
+          return true;
+        hashSet.Add(hash);
+      }
+      return false;
+    }
+    public static Tuple<FlatPathTable,CollisionResolver> Create(List<FSNode> nodes)
+    {
+      var hashMap = new SortedDictionary<uint, uint>();
+      var nodeMap = new Dictionary<uint, List<FSNode>>();
+      bool collision = false;
+      foreach (var n in nodes)
+      {
+        var hash = HashFunction(n.FullPath());
+        if (hashMap.ContainsKey(hash))
+        {
+          hashMap[hash] = 0x80000000;
+          nodeMap[hash].Add(n);
+          collision = true;
+        }
+        else
+        {
+          hashMap[hash] = n.ino.Number | (n is FSDir ? 0x20000000u : 0u);
+          nodeMap[hash] = new List<FSNode>();
+          nodeMap[hash].Add(n);
+        }
+      }
+      if(!collision)
+      {
+        return Tuple.Create(new FlatPathTable(hashMap), (CollisionResolver)null);
+      }
+
+      uint offset = 0;
+      var colEnts = new List<List<PfsDirent>>();
+      foreach(var kv in hashMap.Where(kv => kv.Value == 0x80000000).ToList())
+      {
+        hashMap[kv.Key] = 0x80000000 | offset;
+        var entList = new List<PfsDirent>();
+        colEnts.Add(entList);
+        foreach(var node in nodeMap[kv.Key])
+        {
+          var d = new PfsDirent()
+          {
+            InodeNumber = node.ino.Number,
+            Type = node is FSDir ? DirentType.Directory : DirentType.File,
+            Name = node.FullPath(),
+          };
+          entList.Add(d);
+          offset += (uint)d.EntSize;
+        }
+        offset += 0x18;
+      }
+      return Tuple.Create(new FlatPathTable(hashMap), new CollisionResolver(colEnts));
+    }
+
     private SortedDictionary<uint, uint> hashMap;
 
     public int Size => hashMap.Count * 8;
@@ -19,16 +82,9 @@ class FlatPathTable
     /// Construct a flat_path_table out of the given filesystem nodes.
     /// </summary>
     /// <param name="nodes"></param>
-    public FlatPathTable(List<FSNode> nodes)
+    public FlatPathTable(SortedDictionary<uint, uint> hashMap)
     {
-      hashMap = new SortedDictionary<uint, uint>();
-      foreach (var n in nodes)
-      {
-        var hash = HashFunction(n.FullPath());
-        if (hashMap.ContainsKey(hash))
-          Console.WriteLine("Warning: hash collisions not yet handled.");
-        hashMap[hash] = n.ino.Number | (n is FSDir ? 0x20000000u : 0u);
-      }
+      this.hashMap = hashMap;
     }
 
     /// <summary>
@@ -57,4 +113,37 @@ private static uint HashFunction(string name)
       return hash;
     }
   }
+
+  public class CollisionResolver
+  {
+    public int Size { get; }
+
+    public CollisionResolver(List<List<PfsDirent>> ents)
+    {
+      Entries = ents;
+      var size = 0;
+      foreach(var l in ents)
+      {
+        foreach(var e in l)
+        {
+          size += e.EntSize;
+        }
+        size += 0x18;
+      }
+      Size = size;
+    }
+
+    private List<List<PfsDirent>> Entries;
+    public void WriteToStream(Stream s)
+    {
+      foreach(var d in Entries)
+      {
+        foreach(var e in d)
+        {
+          e.WriteToStream(s);
+        }
+        s.Position += 0x18;
+      }
+    }
+  }
 }

LibOrbisPkg/PFS/PFSBuilder.cs

@@ -21,15 +21,14 @@ public class PfsBuilder
     private List<inode> inodes;
     private List<PfsDirent> super_root_dirents;
 
-    private inode super_root_ino, fpt_ino;
-
-    private FSDir root;
+    private inode super_root_ino, fpt_ino, cr_ino;
 
     private List<FSDir> allDirs;
     private List<FSFile> allFiles;
     private List<FSNode> allNodes;
 
     private FlatPathTable fpt;
+    private CollisionResolver colResolver;
 
     private PfsProperties properties;
 
@@ -97,20 +96,21 @@ void Setup()
       inodes = new List<inode>();
 
       Log("Setting up filesystem structure...");
-      SetupRootStructure();
-      allDirs = root.GetAllChildrenDirs();
-      allFiles = root.GetAllChildrenFiles().Where(f => f.Parent?.name != "sce_sys" || !PKG.EntryNames.NameToId.ContainsKey(f.name)).ToList();
+      allDirs = properties.root.GetAllChildrenDirs();
+      allFiles = properties.root.GetAllChildrenFiles().Where(f => f.Parent?.name != "sce_sys" || !PKG.EntryNames.NameToId.ContainsKey(f.name)).ToList();
       allNodes = new List<FSNode>(allDirs.OrderBy(d => d.FullPath()).ToList());
       allNodes.AddRange(allFiles);
 
+      SetupRootStructure(FlatPathTable.HasCollision(allNodes));
+
       Log($"Creating inodes ({allDirs.Count} dirs and {allFiles.Count} files)...");
       addDirInodes();
       addFileInodes();
 
-      fpt = new FlatPathTable(allNodes);
+      (fpt, colResolver) = FlatPathTable.Create(allNodes);
 
       Log("Calculating data block layout...");
-      allNodes.Insert(0, root);
+      allNodes.Insert(0, properties.root);
       CalculateDataBlockLayout();
     }
 
@@ -121,10 +121,18 @@ private void WriteData(Stream stream)
       WriteInodes(stream);
       WriteSuperrootDirents(stream);
 
-      var fpt_file = new FSFile(s => fpt.WriteToStream(s), "flat_path_table", fpt.Size);
-      fpt_file.ino = fpt_ino;
-      allNodes.Insert(0, fpt_file);
-
+      allNodes.Insert(0, new FSFile(s => fpt.WriteToStream(s), "flat_path_table", fpt.Size)
+      {
+        ino = fpt_ino
+      });
+      if (colResolver != null)
+      {
+        allNodes.Insert(1, new FSFile(s => colResolver.WriteToStream(s), "collision_resolver", colResolver.Size)
+        {
+          ino = cr_ino
+        });
+      }
+
       for (var x = 0; x < allNodes.Count; x++)
       {
         var f = allNodes[x];
@@ -265,7 +273,7 @@ public void WriteImage(Stream stream)
     /// </summary>
     void addDirInodes()
     {
-      inodes.Add(root.ino);
+      inodes.Add(properties.root.ino);
       foreach (var dir in allDirs.OrderBy(x => x.FullPath()))
       {
         var ino = MakeInode(
@@ -343,6 +351,7 @@ long inoNumberToOffset(uint number, int db = 0)
     /// </summary>
     void CalculateDataBlockLayout()
     {
+      // TODO: Consolidate of all this duplicate code
       if (properties.Sign)
       {
         // Include the header block in the total count
@@ -463,8 +472,23 @@ void CalculateDataBlockLayout()
 
         for (int i = 1; i < fpt_ino.Blocks && i < 12; i++)
           fpt_ino.SetDirectBlock(i, (int)hdr.Ndblock++);
-        // DATs I've found include an empty block after the FPT
-        hdr.Ndblock++;
+
+        // DATs I've found include an empty block after the FPT if there's no collision resolver
+        if(cr_ino == null)
+        {
+          hdr.Ndblock++;
+        }
+        else
+        {
+          // collision resolver
+          cr_ino.SetDirectBlock(0, (int)hdr.Ndblock++);
+          cr_ino.Size = colResolver.Size;
+          cr_ino.SizeCompressed = colResolver.Size;
+          cr_ino.Blocks = (uint)CeilDiv(colResolver.Size, hdr.BlockSize);
+
+          for (int i = 1; i < cr_ino.Blocks && i < 12; i++)
+            cr_ino.SetDirectBlock(i, (int)hdr.Ndblock++);
+        }
 
         // Calculate length of all dirent blocks
         foreach (var n in allNodes)
@@ -521,26 +545,36 @@ inode MakeInode(InodeMode Mode, uint Blocks, long Size = 0, long SizeCompressed
     /// Creates inodes and dirents for superroot, flat_path_table, and uroot.
     /// Also, creates the root node for the FS tree.
     /// </summary>
-    void SetupRootStructure()
+    void SetupRootStructure(bool hasCollision)
     {
+      var inodeNum = 0u;
       inodes.Add(super_root_ino = MakeInode(
         Mode: InodeMode.dir | inode.RXOnly,
         Blocks: 1,
         Size: 65536,
         SizeCompressed: 65536,
         Nlink: 1,
-        Number: 0,
+        Number: inodeNum++,
         Flags: InodeFlags.@internal | InodeFlags.@readonly
       ));
       inodes.Add(fpt_ino = MakeInode(
         Mode: InodeMode.file | inode.RXOnly,
         Blocks: 1,
-        Number: 1,
+        Number: inodeNum++,
         Flags: InodeFlags.@internal | InodeFlags.@readonly
       ));
+      if(hasCollision)
+      {
+        inodes.Add(cr_ino = MakeInode(
+          Mode: InodeMode.file | inode.RXOnly,
+          Blocks: 1,
+          Number: inodeNum++,
+          Flags: InodeFlags.@internal | InodeFlags.@readonly
+        ));
+      }
       var uroot_ino = MakeInode(
         Mode: InodeMode.dir | inode.RXOnly,
-        Number: 2,
+        Number: inodeNum++,
         Size: 65536,
         SizeCompressed: 65536,
         Blocks: 1,
@@ -550,17 +584,22 @@ void SetupRootStructure()
 
       super_root_dirents = new List<PfsDirent>
       {
-        new PfsDirent { InodeNumber = 1, Name = "flat_path_table", Type = DirentType.File },
-        new PfsDirent { InodeNumber = 2, Name = "uroot", Type = DirentType.Directory }
+        new PfsDirent { InodeNumber = fpt_ino.Number, Name = "flat_path_table", Type = DirentType.File },
       };
+      if(hasCollision)
+      {
+        super_root_dirents.Add(
+          new PfsDirent { InodeNumber = cr_ino.Number, Name = "collision_resolver", Type = DirentType.File });
+      }
+      super_root_dirents.Add(
+        new PfsDirent { InodeNumber = uroot_ino.Number, Name = "uroot", Type = DirentType.Directory });
 
-      root = properties.root;
-      root.name = "uroot";
-      root.ino = uroot_ino;
-      root.Dirents = new List<PfsDirent>
+      properties.root.name = "uroot";
+      properties.root.ino = uroot_ino;
+      properties.root.Dirents = new List<PfsDirent>
       {
-        new PfsDirent { Name = ".", Type = DirentType.Dot, InodeNumber = 2 },
-        new PfsDirent { Name = "..", Type = DirentType.DotDot, InodeNumber = 2 }
+        new PfsDirent { Name = ".", Type = DirentType.Dot, InodeNumber = uroot_ino.Number },
+        new PfsDirent { Name = "..", Type = DirentType.DotDot, InodeNumber = uroot_ino.Number }
       };
       if(properties.Sign) // HACK: Outer PFS lacks readonly flags
       {

LibOrbisPkgTests/LibOrbisPkgTests.csproj

@@ -49,6 +49,7 @@
     <Reference Include="System.Core" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="PfsTests.cs" />
     <Compile Include="PkgBuildTest.cs" />
     <Compile Include="PkgReadTest.cs" />
     <Compile Include="SfoTest.cs" />

LibOrbisPkgTests/PfsTests.cs

@@ -0,0 +1,41 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using LibOrbisPkg.GP4;
+using LibOrbisPkg.PKG;
+using LibOrbisPkg.PFS;
+using LibOrbisPkg.SFO;
+using LibOrbisPkg.Util;
+using System.IO;
+using System.IO.MemoryMappedFiles;
+
+namespace LibOrbisPkgTests
+{
+  /// <summary>
+  /// Tests functionality of building PKGs
+  /// </summary>
+  [TestClass]
+  public class PfsTests
+  {
+    [TestMethod]
+    public void TestFptHasCollision()
+    {
+      var root = TestHelper.MakeRoot();
+      root.Files.Add(new FSFile(s => s.Position = s.Position, "AO", 0) { Parent = root });
+      root.Files.Add(new FSFile(s => s.Position = s.Position, "B0", 0) { Parent = root });
+      Assert.IsTrue(FlatPathTable.HasCollision(root.GetAllChildren()));
+    }
+
+    [TestMethod]
+    public void TestFptNoCollision()
+    {
+      var root = TestHelper.MakeRoot();
+      root.Files.Add(new FSFile(s => s.Position = s.Position, "AO", 0) { Parent = root });
+      root.Files.Add(new FSFile(s => s.Position = s.Position, "B1", 0) { Parent = root });
+      Assert.IsFalse(FlatPathTable.HasCollision(root.GetAllChildren()));
+    }
+  }
+}