Support archive format version 9 chunk alignment padding (#128)

A change in Unity 6.7 bumps the Unity Archive format to version 9 and introduces
padding - each chunk-based (non-streamed) storage
block is followed by zero padding up to the next 16-byte boundary of the data
section, so that a size change in one chunk no longer shifts the position of
every chunk after it (this aids binary patching).

each StorageBlock now serializes a UInt64 offset giving where its stored bytes
start, relative to the start of the data section. The writer is then free to
change its padding policy without another format bump, and the reader no longer
needs the "streamed blocks are not padded, chunk blocks are" special case.

Parse that offset as the fourth field of each block record, after Flags, when
the signature is UnityFS and the version is 9 or later. Version 8 and earlier
store their blocks contiguously, so their offsets are still accumulated from
the compressed sizes. A v9 archive whose blocks are out of order or overlapping
is rejected, matching the native reader.

Author: SkowronskiAndrew

Archive/ArchiveTool.cs

@@ -163,6 +163,16 @@ public static int PrintSummary(FileInfo filename, OutputFormat format)
             uncompressedDataSize += block.UncompressedSize;
         }
 
+        // The data section can be larger than the sum of the blocks, because the writer may leave
+        // padding between them. Only version 9 and later record where each block starts, so this is
+        // the only version where padding is detectable (and where it is currently written).
+        long paddingSize = 0;
+        if (blocks.Length > 0)
+        {
+            var lastBlock = blocks[blocks.Length - 1];
+            paddingSize = lastBlock.Offset + lastBlock.CompressedSize - dataSize;
+        }
+
         // Determine the compression algorithm by finding the first block that uses compression.
         // Individual blocks may be stored uncompressed even when compression is enabled, because
         // compression is skipped when it provides no size reduction. So the first compressed block
@@ -193,6 +203,7 @@ public static int PrintSummary(FileInfo filename, OutputFormat format)
                 unityVersion = header.UnityVersion,
                 fileSize = header.Size,
                 dataSize = dataSize,
+                blockPaddingSize = paddingSize,
                 uncompressedDataSize = uncompressedDataSize,
                 compressionRatio = Math.Round(compressionRatio, 2),
                 compression = compression,
@@ -208,6 +219,8 @@ public static int PrintSummary(FileInfo filename, OutputFormat format)
             Console.WriteLine($"{"Unity Version",-30} {header.UnityVersion}");
             Console.WriteLine($"{"File Size",-30} {header.Size:N0} bytes");
             Console.WriteLine($"{"Data Size",-30} {dataSize:N0} bytes");
+            if (paddingSize > 0)
+                Console.WriteLine($"{"Block Padding Size",-30} {paddingSize:N0} bytes");
             Console.WriteLine($"{"Uncompressed Data Size",-30} {uncompressedDataSize:N0} bytes");
             Console.WriteLine($"{"Compression Ratio",-30} {compressionRatio:F2}x");
             Console.WriteLine($"{"Compression",-30} {compression}");
@@ -263,10 +276,10 @@ static string FormatCompressionType(int compressionType)
 
     static readonly (uint bit, string name)[] KnownArchiveFlags =
     {
-        (0x40,  "BlocksAndDirectoryInfoCombined"),
-        (0x80,  "BlocksInfoAtTheEnd"),
-        (0x100, "OldWebPluginCompatibility"),
-        (0x200, "BlockInfoNeedPaddingAtStart"),
+        (ArchiveFlags.BlocksAndDirectoryInfoCombined, "BlocksAndDirectoryInfoCombined"),
+        (ArchiveFlags.BlocksInfoAtTheEnd,             "BlocksInfoAtTheEnd"),
+        (ArchiveFlags.OldWebPluginCompatibility,      "OldWebPluginCompatibility"),
+        (ArchiveFlags.BlockInfoNeedPaddingAtStart,    "BlockInfoNeedPaddingAtStart"),
     };
 
     static string[] GetArchiveFlagNames(uint flagBits)

TestCommon/Data/LeadingEdgeBuilds/AGENTS.md

@@ -6,7 +6,8 @@ The LeadingEdge build scripts regenerate this folder directly, so to update it,
 
 ## Layout
 
-* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle.
+* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle. LZMA compressed, so each archive has a single data block.
+* `AssetBundlesLz4/` - the same bundle layout built with chunk-based (LZ4) compression, giving archives with multiple data blocks. The writer leaves padding between the blocks, so these exercise the `StorageBlock.Offset` field added in archive format version 9.
 * `ContentDirectory/` - the Content Directory build: content (`.cf`) files, `.resource` files and the build manifest.
 * `BuildReport-AssetBundles/LastBuild.buildreport` - the AssetBundle build report.
 * `BuildReport-ContentDirectory/` - the Content Directory build report folder, including `ContentLayout.json`.

TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6

@@ -0,0 +1,21 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.7.0b2
+CRC: 1943531319
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 5e3dcfafd0199141af65300e6ff5a8fb
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 9a2ca7bdbd1871f7131daf57de908e0c
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: 929d8f9d46e767e1fc66294cd024c1fa
+HashAppended: 0
+ClassTypes:
+- Class: 83
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers: []
+Assets:
+- Assets/Audio/6.mp3
+Dependencies: []

TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest

@@ -0,0 +1,33 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.7.0b2
+CRC: 3869125346
+HashAppended: 0
+AssetBundleManifest:
+  AssetBundleInfos:
+    Info_0:
+      Name: assetbundleroot
+      Dependencies:
+        Dependency_0: directaudioclipreference
+        Dependency_1: singleaudioclipdirectreference
+        Dependency_2: serializationdemo
+    Info_1:
+      Name: directaudioclipreference
+      Dependencies:
+        Dependency_0: 6
+        Dependency_1: a
+    Info_2:
+      Name: singleaudioclipdirectreference
+      Dependencies:
+        Dependency_0: a
+    Info_3:
+      Name: serializationdemo
+      Dependencies: {}
+    Info_4:
+      Name: 6
+      Dependencies: {}
+    Info_5:
+      Name: a
+      Dependencies: {}
+    Info_6:
+      Name: scenes
+      Dependencies: {}

TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4

@@ -0,0 +1,21 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.7.0b2
+CRC: 1309966254
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: 515b741faeaecc91b64c7a8b6504b338
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 9a2ca7bdbd1871f7131daf57de908e0c
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: aeb2216eeb5ccfe1cb3f81e62f704746
+HashAppended: 0
+ClassTypes:
+- Class: 83
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers: []
+Assets:
+- Assets/Audio/a.mp3
+Dependencies: []

TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest

@@ -0,0 +1,34 @@
+ManifestFileVersion: 0
+UnityVersion: 6000.7.0b2
+CRC: 1322119296
+Hashes:
+  AssetFileHash:
+    serializedVersion: 2
+    Hash: c174946e4bcb463b2dd90535ff0dcfe5
+  TypeTreeHash:
+    serializedVersion: 2
+    Hash: 819240184df2798191fd853444ad58c4
+  IncrementalBuildHash:
+    serializedVersion: 2
+    Hash: cf13ad57198a935f56c518df40c037d0
+HashAppended: 0
+ClassTypes:
+- Class: 114
+  Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3}
+- Class: 114
+  Script: {fileID: 11500000, guid: 8623c5efbb626994da80931050f0aba0, type: 3}
+- Class: 114
+  Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3}
+- Class: 115
+  Script: {instanceID: 0}
+SerializeReferenceClassIdentifiers:
+- AssemblyName: Assembly-CSharp
+  ClassName: SerializationDemo/SerializedData
+- AssemblyName: UnityEngine.CoreModule
+  ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2
+Assets:
+- Assets/ScriptableObjects/AssetBundleRoot.asset
+Dependencies:
+- C:/UnitySrc/UnityDataTools_Cursor/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference
+- C:/UnitySrc/UnityDataTools_Cursor/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo
+- C:/UnitySrc/UnityDataTools_Cursor/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference