Add test with a lot of files (5000) in one directory, fix bug exposed…

… by said test
maxton · Jul 29, 2019 · baffa88 · baffa88
1 parent 7bce937
commit baffa88
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 14 deletions.
diff --git a/LibOrbisPkg/PFS/PFSBuilder.cs b/LibOrbisPkg/PFS/PFSBuilder.cs
@@ -271,7 +271,7 @@ void addDirInodes()
       foreach (var dir in allDirs)
       {
         var ino = MakeInode(
-          Mode: InodeMode.dir | InodeMode.rx_only,
+          Mode: InodeMode.dir | inode.RXOnly,
           Number: (uint)inodes.Count,
           Blocks: 1,
           Size: 65536,
@@ -297,7 +297,7 @@ void addFileInodes()
       foreach (var file in allFiles.OrderBy(x => x.FullPath()))
       {
         var ino = MakeInode(
-          Mode: InodeMode.file | InodeMode.rx_only,
+          Mode: InodeMode.file | inode.RXOnly,
           Size: file.Size,
           SizeCompressed: file.CompressedSize,
           Number: (uint)inodes.Count,
@@ -440,7 +440,8 @@ long inoNumberToOffset(uint number, int db = 0)
         hdr.InodeBlockSig.SetTime(properties.FileTime);
         for (var i = 1; i < hdr.DinodeBlockCount; i++)
         {
-          hdr.InodeBlockSig.SetDirectBlock(i, -1);
+          if(i < 12)
+            hdr.InodeBlockSig.SetDirectBlock(i, -1);
           hdr.Ndblock++;
         }
         super_root_ino.SetDirectBlock(0, (int)hdr.Ndblock);
@@ -515,7 +516,7 @@ inode MakeInode(InodeMode Mode, uint Blocks, long Size = 0, long SizeCompressed
     void SetupRootStructure()
     {
       inodes.Add(super_root_ino = MakeInode(
-        Mode: InodeMode.dir | InodeMode.rx_only,
+        Mode: InodeMode.dir | inode.RXOnly,
         Blocks: 1,
         Size: 65536,
         SizeCompressed: 65536,
@@ -524,13 +525,13 @@ void SetupRootStructure()
         Flags: InodeFlags.@internal | InodeFlags.@readonly
       ));
       inodes.Add(fpt_ino = MakeInode(
-        Mode: InodeMode.file | InodeMode.rx_only,
+        Mode: InodeMode.file | inode.RXOnly,
         Blocks: 1,
         Number: 1,
         Flags: InodeFlags.@internal | InodeFlags.@readonly
       ));
       var uroot_ino = MakeInode(
-        Mode: InodeMode.dir | InodeMode.rx_only,
+        Mode: InodeMode.dir | inode.RXOnly,
         Number: 2,
         Size: 65536,
         SizeCompressed: 65536,

diff --git a/LibOrbisPkg/PFS/PfsStructs.cs b/LibOrbisPkg/PFS/PfsStructs.cs
@@ -137,8 +137,6 @@ public enum InodeMode : ushort
     u_execute = 256,
     dir = 16384,
     file = 32768,
-    rx_only = 0x16D,
-    rwx = 0x1FF
   }
 
   /// <summary>
@@ -172,6 +170,11 @@ public enum InodeFlags : uint
   /// </summary>
   public abstract class inode
   {
+    public const InodeMode RXOnly =
+      InodeMode.o_read | InodeMode.o_execute |
+      InodeMode.g_read | InodeMode.g_execute |
+      InodeMode.u_read | InodeMode.u_execute;
+
     public inode()
     {
       SetTime((long)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds);
@@ -517,12 +520,18 @@ public string Name
       {
         name = value;
         NameLength = name.Length;
-        EntSize = NameLength + 17;
-        if (EntSize % 8 != 0)
-          EntSize += 8 - (EntSize % 8);
+        EntSize = CalculateEntSize();
       }
     }
 
+    public int CalculateEntSize()
+    {
+      var EntSize = NameLength + 17;
+      if (EntSize % 8 != 0)
+        EntSize += 8 - (EntSize % 8);
+      return EntSize;
+    }
+
     private string name;
 
     public void WriteToStream(Stream s)

diff --git a/LibOrbisPkg/Util/MemoryMapped.cs b/LibOrbisPkg/Util/MemoryMapped.cs
@@ -56,8 +56,17 @@ class MemoryAccessor : IMemoryAccessor
     private long offset;
     public MemoryAccessor(IMemoryReader mr, long offset = 0)
     {
-      this.offset = offset;
-      reader = mr;
+      // Efficient nesting of MemoryAccessors: eliminating long call chains to the base IMemoryReader
+      if(mr is MemoryAccessor ma)
+      {
+        this.offset = offset + ma.offset;
+        this.reader = ma.reader;
+      }
+      else
+      {
+        this.offset = offset;
+        this.reader = mr;
+      }
     }
     public void Dispose() { }
     public void Read<T>(long pos, out T value) where T : struct

diff --git a/LibOrbisPkgTests/PkgBuildTest.cs b/LibOrbisPkgTests/PkgBuildTest.cs
@@ -9,6 +9,7 @@
 using LibOrbisPkg.PKG;
 using LibOrbisPkg.PFS;
 using LibOrbisPkg.SFO;
+using LibOrbisPkg.Util;
 using System.IO;
 using System.IO.MemoryMappedFiles;
 
@@ -74,6 +75,41 @@ public void EmptyPkgMMFile()
       }
     }
 
+    [TestMethod]
+    public void ManyFilePkg()
+    {
+      const int NumFiles = 5000;
+      using (var pkgFile = new TempFile())
+      {
+        var buf = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+        Action<Stream> fileWriter = s => s.Write(buf, 0, 16);
+        var rootDir = TestHelper.MakeRoot();
+        var filesDir = new FSDir() { Parent = rootDir, name = "files" };
+        rootDir.Dirs.Add(filesDir);
+        for(int i = 0; i < NumFiles; i++)
+        {
+          filesDir.Files.Add(new FSFile(fileWriter, $"file_{i}", 16) { Parent = filesDir });
+        }
+        new PkgBuilder(TestHelper.MakeProperties(RootDir: rootDir)).Write(pkgFile.Path, Console.WriteLine);
+        int foundFiles = 0;
+        TestHelper.OpenPkgFilesystem(pkgFile.Path, innerPfs =>
+          {
+            var testBuf = new byte[16];
+            foreach(var f in innerPfs.GetURoot().GetAllFiles())
+            {
+              foundFiles++;
+              using (var view = f.GetView())
+              {
+                view.Read(0, testBuf, 0, 16);
+                CollectionAssert.AreEqual(buf, testBuf, $"Expected {buf.AsHexCompact()}, got {testBuf.AsHexCompact()}");
+              }
+            }
+          });
+
+        Assert.AreEqual(NumFiles, foundFiles);
+      }
+    }
+
     /// <summary>
     /// Tests that a PKG passes all validation checks.
     /// </summary>

diff --git a/PS4PFS.bt b/PS4PFS.bt
@@ -212,13 +212,15 @@ typedef struct {
 
 
 typedef struct {
+    local int64 start = FTell();
     int32 ino;
     int32 type;
     int32 namelen;
     int32 entsize;
     char name[];
     if((FTell() & 7) != 0)
         char space[(FTell() + 7 & (~7)) - FTell()];
+    FSeek(start + entsize);
 } dirent<read=DIRENTRD>;
 
 string DIRENTRD(dirent& di){
@@ -294,7 +296,7 @@ typedef struct {
 
                 cont = true;
                   while(cont == true){
-                    dirent d;
+                    dirent d<optimize=false>;
 
                     cont = d.ino != 0;
                   }