From 518d80685674897e26bcd41597d1ad4087a3a2e9 Mon Sep 17 00:00:00 2001 From: George Wu Date: Sun, 25 Feb 2018 17:42:39 +0100 Subject: [PATCH 1/4] ACB serialization fixes. --- .../Serialization/AcbSerializer.Internal.cs | 70 ++++++++++++++----- .../Serialization/AcbSerializer.cs | 11 +-- .../Serialization/ArrayExtensions.cs | 6 +- .../Serialization/SerializationHelper.cs | 55 ++++++++++++--- .../Serialization/UtfFieldAttribute.cs | 6 +- .../Serialization/UtfFieldImage.cs | 9 +-- .../Serialization/UtfRowBase.cs | 5 +- .../Serialization/UtfTableAttribute.cs | 4 +- 8 files changed, 116 insertions(+), 50 deletions(-) diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.Internal.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.Internal.cs index 0b3e6dd..6f126eb 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.Internal.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.Internal.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -6,57 +6,88 @@ namespace DereTore.Exchange.Archive.ACB.Serialization { public partial class AcbSerializer { - private byte[] GetTableBytes(T[] tableRows) where T : UtfRowBase { + /// + /// Get the unaligned byte array image of a table (UTF row). + /// Returned array should be aligned by . + /// + /// Rows in this table. + /// + private byte[] GetTableBytes(UtfRowBase[] tableRows) { var tableImage = PrepareTable(tableRows); + byte[] buffer; + using (var memory = new MemoryStream()) { tableImage.WriteTo(memory); - //memory.Capacity = (int)AcbHelper.RoundUpToAlignment((int)memory.Length, Alignment); - buffer = new byte[memory.Length]; - //memory.GetBuffer().CopyTo(buffer, 0); - Array.Copy(memory.GetBuffer(), buffer, buffer.Length); + buffer = memory.ToArray(); } + return buffer; } - private UtfTableImage PrepareTable(T[] tableRows) where T : UtfRowBase { - var tableName = GetTableName(tableRows); + private UtfTableImage PrepareTable(UtfRowBase[] tableRows) { + if (tableRows.Length < 1) { + throw new ArgumentException("There should be at least one row in a table.", nameof(tableRows)); + } + + var tableRowTypes = tableRows.Select(row => row.GetType()).ToArray(); + + if (tableRowTypes.Length > 1) { + for (var i = 1; i < tableRowTypes.Length; ++i) { + if (tableRowTypes[i] != tableRowTypes[i - 1]) { + throw new ArgumentException("All rows must have the same CLR type."); + } + } + } + + var tableName = GetTableName(tableRows, tableRowTypes[0]); var tableImage = new UtfTableImage(tableName, Alignment); + foreach (var tableRow in tableRows) { var targetMembers = SerializationHelper.GetSearchTargetFieldsAndProperties(tableRow); var tableImageRow = new List(); + tableImage.Rows.Add(tableImageRow); + // TODO: Save misc data first, then the tables. foreach (var member in targetMembers) { var fieldInfo = member.FieldInfo; var fieldType = fieldInfo.FieldType; var fieldAttribute = member.FieldAttribute; + CheckFieldType(fieldType); + var fieldValue = member.FieldInfo.GetValue(tableRow); + var fieldImage = new UtfFieldImage { Order = fieldAttribute.Order, Name = string.IsNullOrEmpty(fieldAttribute.FieldName) ? fieldInfo.Name : fieldAttribute.FieldName, Storage = fieldAttribute.Storage, }; + // Empty tables are treated as empty data. if (IsTypeRowList(fieldType) && fieldValue != null && ((UtfRowBase[])fieldValue).Length > 0) { var tableBytes = GetTableBytes((UtfRowBase[])fieldValue); + fieldImage.SetValue(tableBytes); fieldImage.IsTable = true; } else if (fieldType == typeof(byte[]) && member.ArchiveAttribute != null) { var files = new List { (byte[])fieldValue }; + var archiveBytes = SerializationHelper.GetAfs2ArchiveBytes(files.AsReadOnly(), Alignment); + fieldImage.SetValue(archiveBytes); fieldImage.IsTable = true; } else { if (fieldValue == null) { - fieldImage.SetNullValue(MapFromRawType(fieldType)); + fieldImage.SetNullValue(MapUtfColumeTypeFromRawType(fieldType)); } else { fieldImage.SetValue(fieldValue); } } + tableImageRow.Add(fieldImage); } } @@ -73,31 +104,32 @@ private static bool IsTypeRowList(Type type) { return type.IsArray && type.HasElementType && type.GetElementType().IsSubclassOf(typeof(UtfRowBase)); } - private static ColumnType MapFromRawType(Type rawType) { + private static ColumnType MapUtfColumeTypeFromRawType(Type rawType) { if (IsTypeRowList(rawType)) { return ColumnType.Data; } + var index = SupportedTypes.IndexOf(rawType); + return (ColumnType)index; } - private static string GetTableName(T[] tableRows) where T : UtfRowBase { + private static string GetTableName(UtfRowBase[] tableRows, Type tableType) { if (tableRows == null) { throw new ArgumentNullException(nameof(tableRows)); } - if (tableRows.Length < 1) { - throw new ArgumentException("There should be at least one row in a table.", nameof(tableRows)); - } - // Assuming all the rows are of the same type. - var tableType = tableRows[0].GetType(); - var tableAttributes = tableType.GetCustomAttributes(typeof(UtfTableAttribute), false); - if (tableAttributes.Length == 1) { - return ((UtfTableAttribute)tableAttributes[0]).Name; + + var tableAttribute = SerializationHelper.GetCustomAttribute(tableType); + + if (!string.IsNullOrEmpty(tableAttribute?.Name)) { + return tableAttribute.Name; } else { var s = tableType.Name; + if (s.EndsWith("Table")) { s = s.Substring(0, s.Length - 5); } + return s; } } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.cs index 7bd3cbb..c6fa6d1 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/AcbSerializer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using DereTore.Common; @@ -9,22 +9,23 @@ public AcbSerializer() { Alignment = 32; } - public void Serialize(T[] tableRows, Stream serializationStream) where T : UtfRowBase { + public void Serialize(UtfRowBase[] tableRows, Stream serializationStream) { if (tableRows == null) { throw new ArgumentNullException(nameof(tableRows)); } + var tableData = GetTableBytes(tableRows).RoundUpTo(Alignment); + serializationStream.WriteBytes(tableData); } public uint Alignment { - get { - return _alignment; - } + get { return _alignment; } set { if (value <= 0 || value % 16 != 0) { throw new ArgumentException("Alignment should be a positive integer, also a multiple of 16.", nameof(value)); } + _alignment = value; } } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/ArrayExtensions.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/ArrayExtensions.cs index 4360b5f..ea1d7e7 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/ArrayExtensions.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/ArrayExtensions.cs @@ -1,17 +1,21 @@ -namespace DereTore.Exchange.Archive.ACB.Serialization { +namespace DereTore.Exchange.Archive.ACB.Serialization { internal static class ArrayExtensions { public static byte[] RoundUpTo(this byte[] data, int alignment) { var newLength = AcbHelper.RoundUpToAlignment(data.Length, alignment); var buffer = new byte[newLength]; + data.CopyTo(buffer, 0); + return buffer; } public static byte[] RoundUpTo(this byte[] data, uint alignment) { var newLength = AcbHelper.RoundUpToAlignment(data.Length, alignment); var buffer = new byte[newLength]; + data.CopyTo(buffer, 0); + return buffer; } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs index 2c2c52b..d59a8fb 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; @@ -11,9 +11,11 @@ internal static class SerializationHelper { public static uint RoundUpAsTable(uint value, uint alignment) { // This action seems weird. But it does exist (see Cue table in CGSS song_1001[oneshin]), I don't know why. value = AcbHelper.RoundUpToAlignment(value, 4); + if (value % alignment == 0) { value += alignment; } + return AcbHelper.RoundUpToAlignment(value, alignment); } @@ -21,17 +23,28 @@ public static MemberAbstract[] GetSearchTargetFieldsAndProperties(UtfRowBase tab var type = tableObject.GetType(); var objFields = type.GetFields(BindingFlags.Instance | BindingFlags.DeclaredOnly | BindingFlags.Public | BindingFlags.NonPublic); var validDescriptors = new List(); + var lastOrder = -1; + foreach (var field in objFields) { - var caField = field.GetCustomAttributes(typeof(UtfFieldAttribute), false); + var utfFieldAttribute = GetCustomAttribute(field); + // It is a field that needs serialization. - if (caField.Length == 1) { - var caArchive = field.GetCustomAttributes(typeof(Afs2ArchiveAttribute), false); - var ca1 = caField[0] as UtfFieldAttribute; - var ca2 = caArchive.Length == 1 ? caArchive[0] as Afs2ArchiveAttribute : null; - validDescriptors.Add(new MemberAbstract(field, ca1, ca2)); + if (utfFieldAttribute != null) { + var afs2ArchiveAttribute = GetCustomAttribute(field); + + if (utfFieldAttribute.Order < 0) { + ++lastOrder; + utfFieldAttribute.Order = lastOrder; + } else { + lastOrder = utfFieldAttribute.Order; + } + + validDescriptors.Add(new MemberAbstract(field, utfFieldAttribute, afs2ArchiveAttribute)); } } - validDescriptors.Sort((d1, d2) => d1.FieldAttribute.Order - d2.FieldAttribute.Order); + + validDescriptors.Sort((d1, d2) => d1.FieldAttribute.Order.CompareTo(d2.FieldAttribute.Order)); + return validDescriptors.ToArray(); } @@ -39,15 +52,37 @@ public static byte[] GetAfs2ArchiveBytes(ReadOnlyCollection files, uint if (files.Count == 0) { return new byte[0]; } + byte[] buffer; + using (var memory = new MemoryStream()) { WriteAfs2ArchiveToStream(files, memory, alignment); - buffer = new byte[memory.Length]; - Array.Copy(memory.GetBuffer(), buffer, buffer.Length); + buffer = memory.ToArray(); } + return buffer; } + public static T GetCustomAttribute(Type type, bool inherit = false) where T : Attribute { + var attr = type.GetCustomAttributes(typeof(T), inherit); + + if (attr.Length == 0) { + return null; + } else { + return attr[0] as T; + } + } + + public static T GetCustomAttribute(FieldInfo fieldInfo, bool inherit = false) where T : Attribute { + var attr = fieldInfo.GetCustomAttributes(typeof(T), inherit); + + if (attr.Length == 0) { + return null; + } else { + return attr[0] as T; + } + } + private static void WriteAfs2ArchiveToStream(ReadOnlyCollection files, Stream stream, uint alignment) { var fileCount = (uint)files.Count; if (files.Count >= ushort.MaxValue) { diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldAttribute.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldAttribute.cs index 5cfe900..a65a7ce 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldAttribute.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldAttribute.cs @@ -1,16 +1,16 @@ -using System; +using System; namespace DereTore.Exchange.Archive.ACB.Serialization { [AttributeUsage(AttributeTargets.Field)] public sealed class UtfFieldAttribute : Attribute { - public UtfFieldAttribute(int order, ColumnStorage storage = ColumnStorage.PerRow, string fieldName = null) { + public UtfFieldAttribute(int order = -1, ColumnStorage storage = ColumnStorage.PerRow, string fieldName = null) { Order = order; Storage = storage; FieldName = fieldName; } - public int Order { get; } + public int Order { get; internal set; } public string FieldName { get; } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs index 5ee8083..789de8c 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs @@ -1,13 +1,10 @@ -using System; +using System; using System.IO; using DereTore.Common; namespace DereTore.Exchange.Archive.ACB.Serialization { internal sealed class UtfFieldImage : IComparable { - internal UtfFieldImage() { - } - public ColumnStorage Storage { get; set; } public ColumnType Type { get; set; } public string Name { get; set; } @@ -139,11 +136,11 @@ public void WriteValueTo(Stream stream) { } public int CompareTo(UtfFieldImage other) { - return this.Order - other.Order; + return Order.CompareTo(other.Order); } public override string ToString() { - return $"UtfFieldImage {{{Name}}}"; + return $"UtfFieldImage {{{Name}}} ({Type})"; } } } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfRowBase.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfRowBase.cs index e2dc48a..279bba8 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfRowBase.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfRowBase.cs @@ -1,8 +1,5 @@ -namespace DereTore.Exchange.Archive.ACB.Serialization { +namespace DereTore.Exchange.Archive.ACB.Serialization { public abstract class UtfRowBase { - protected UtfRowBase() { - } - } } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableAttribute.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableAttribute.cs index d4e61ee..0a9496c 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableAttribute.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableAttribute.cs @@ -1,7 +1,7 @@ -using System; +using System; namespace DereTore.Exchange.Archive.ACB.Serialization { - [AttributeUsage(AttributeTargets.Class)] + [AttributeUsage(AttributeTargets.Class, Inherited = false)] public sealed class UtfTableAttribute : Attribute { public UtfTableAttribute(string name) { From 14dd112afac565cb801c23cb55e9c0b26131f718 Mon Sep 17 00:00:00 2001 From: George Wu Date: Sun, 25 Feb 2018 22:10:04 +0100 Subject: [PATCH 2/4] Test table alignment. --- Apps/Hcaenc/Program.cs | 31 +++++++++++++--- .../Serialization/SerializationHelper.cs | 6 ++-- .../Serialization/UtfTableImage.Output.cs | 26 ++++++++++---- .../UtfTableImage.Preprocessor.cs | 36 +++++++++++++------ .../UtfTableImage.SizeCalculation.cs | 24 ++++++++++--- 5 files changed, 94 insertions(+), 29 deletions(-) diff --git a/Apps/Hcaenc/Program.cs b/Apps/Hcaenc/Program.cs index b5bdb25..d47560f 100644 --- a/Apps/Hcaenc/Program.cs +++ b/Apps/Hcaenc/Program.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.IO; using System.Runtime.InteropServices; namespace DereTore.Apps.Hcaenc { @@ -9,19 +10,39 @@ private static int Main(string[] args) { Console.WriteLine(UnsupportedBuildMessage); return -2; } - if (args.Length != 2) { + + if (args.Length < 1) { Console.WriteLine(HelpMessage); return -1; } - int quality = 1, cutoff = 0; + + var inputFile = args[0]; + + inputFile = Path.GetFullPath(inputFile); + + string outputFile; + + if (args.Length >= 2) { + outputFile = args[1]; + } else { + var inputFileInfo = new FileInfo(inputFile); + var inputFileDir = inputFileInfo.DirectoryName; + outputFile = Path.Combine(inputFileDir, inputFileInfo.Name.Substring(0, inputFileInfo.Name.Length - inputFileInfo.Extension.Length) + ".hca"); + } + + Console.WriteLine("Encoding {0} to {1} ...", inputFile, outputFile); + + // Quality = 3 (~128 Kbps) for MLTD, 1 (~256 Kbps) for CGSS + int quality = 3, cutoff = 0; ulong key = 0; - return hcaencEncodeToFile(args[0], args[1], quality, cutoff, key); + + return hcaencEncodeToFile(inputFile, outputFile, quality, cutoff, key); } [DllImport("hcaenc_lite", CallingConvention = CallingConvention.StdCall)] private static extern int hcaencEncodeToFile([MarshalAs(UnmanagedType.LPStr)] string lpstrInputFile, [MarshalAs(UnmanagedType.LPStr)] string lpstrOutputFile, int nQuality, int nCutoff, ulong ullKey); - private static readonly string HelpMessage = "Usage: hcaenc.exe "; + private static readonly string HelpMessage = "Usage: hcaenc.exe []"; private static readonly string UnsupportedBuildMessage = "hcaenc only has 32-bit version due to the limits of hcaenc_lite.dll."; } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs index d59a8fb..1eaf5b2 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs @@ -12,9 +12,9 @@ public static uint RoundUpAsTable(uint value, uint alignment) { // This action seems weird. But it does exist (see Cue table in CGSS song_1001[oneshin]), I don't know why. value = AcbHelper.RoundUpToAlignment(value, 4); - if (value % alignment == 0) { - value += alignment; - } + //if (value % alignment == 0) { + // value += alignment; + //} return AcbHelper.RoundUpToAlignment(value, alignment); } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Output.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Output.cs index 0aa6195..08421de 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Output.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Output.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using DereTore.Common; @@ -8,16 +8,20 @@ internal sealed partial class UtfTableImage { public void WriteTo(Stream bufferStream) { Rows.ForEach(row => { row.Sort((t1, t2) => { - if (!ReferenceEquals(t1, t2) && t1.Order == t2.Order) { + if (!ReferenceEquals(t1, t2) && t1.Order.Equals(t2.Order)) { throw new InvalidOperationException("Each field must have different order."); } - return t1.Order - t2.Order; + + return t1.Order.CompareTo(t2.Order); }); }); + ProcessStringTable(); ProcessDataFields(); + UtfFieldImage[] orderedDataFieldImages; var header = GetHeader(out orderedDataFieldImages); + WriteHeaderTo(header, bufferStream); SetStringFieldRelocations(header); SetExtraDataFieldRelocations(header, orderedDataFieldImages); @@ -30,7 +34,7 @@ public void WriteTo(Stream bufferStream) { private static void WriteHeaderTo(UtfHeader header, Stream bufferStream) { bufferStream.SeekAndWriteBytes(UtfTable.UtfSignature, 0); bufferStream.SeekAndWriteUInt32BE(header.TableSize - 8, 4); - bufferStream.SeekAndWriteUInt16BE(1, 8); + bufferStream.SeekAndWriteUInt16BE(header.Unknown1, 8); bufferStream.SeekAndWriteUInt16BE((ushort)(header.PerRowDataOffset - 8), 10); bufferStream.SeekAndWriteUInt32BE(header.StringTableOffset - 8, 12); bufferStream.SeekAndWriteUInt32BE(header.ExtraDataOffset - 8, 16); @@ -42,10 +46,13 @@ private static void WriteHeaderTo(UtfHeader header, Stream bufferStream) { private void WriteFieldDescriptorsTo(UtfHeader header, Stream bufferStream) { bufferStream.Seek(SchemaOffset, SeekOrigin.Begin); + var row = Rows[0]; + foreach (var fieldImage in row) { bufferStream.WriteByte((byte)((byte)fieldImage.Storage | (byte)fieldImage.Type)); bufferStream.WriteUInt32BE(fieldImage.NameOffset); + if (fieldImage.Storage == ColumnStorage.Constant || fieldImage.Storage == ColumnStorage.Constant2) { fieldImage.WriteValueTo(bufferStream); } @@ -54,6 +61,7 @@ private void WriteFieldDescriptorsTo(UtfHeader header, Stream bufferStream) { private void WritePerRowDataTo(UtfHeader header, Stream bufferStream) { bufferStream.Seek(header.PerRowDataOffset, SeekOrigin.Begin); + foreach (var row in Rows) { foreach (var fieldImage in row) { if (fieldImage.Storage == ColumnStorage.PerRow) { @@ -66,15 +74,19 @@ private void WritePerRowDataTo(UtfHeader header, Stream bufferStream) { private void WriteStringTableTo(UtfHeader header, Stream bufferStream) { bufferStream.Seek(header.StringTableOffset, SeekOrigin.Begin); bufferStream.WriteBytes(TableNameBytesCache); + var row0 = Rows[0]; + foreach (var fieldImage in row0) { bufferStream.WriteBytes(fieldImage.NameBytesCache); } + foreach (var fieldImage in row0) { if ((fieldImage.Storage == ColumnStorage.Constant || fieldImage.Storage == ColumnStorage.Constant2) && fieldImage.Type == ColumnType.String) { bufferStream.WriteBytes(fieldImage.StringValueBytesCache); } } + foreach (var row in Rows) { foreach (var fieldImage in row) { if (fieldImage.Type == ColumnType.String && fieldImage.Storage == ColumnStorage.PerRow) { @@ -86,11 +98,14 @@ private void WriteStringTableTo(UtfHeader header, Stream bufferStream) { private void WriteExtraDataTo(UtfHeader header, Stream bufferStream) { var baseOffset = header.ExtraDataOffset; + for (var i = 0; i < Rows.Count; ++i) { var row = Rows[i]; + foreach (var fieldImage in row) { if (fieldImage.Type == ColumnType.Data) { var shouldWrite = false; + switch (fieldImage.Storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -99,9 +114,8 @@ private void WriteExtraDataTo(UtfHeader header, Stream bufferStream) { case ColumnStorage.PerRow: shouldWrite = true; break; - default: - break; } + if (shouldWrite && fieldImage.DataValue.Length > 0) { bufferStream.SeekAndWriteBytes(fieldImage.DataValue, baseOffset + fieldImage.DataOffset); } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs index 23dfa1d..78048a4 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Text; using DereTore.Common; @@ -14,7 +14,9 @@ public UtfHeader GetHeader(out UtfFieldImage[] orderedDataFieldImages) { if (Rows.Count < 1) { throw new InvalidOperationException("Rows should not be empty."); } + var fieldCount = Rows[0].Count; + for (var i = 1; i < Rows.Count; ++i) { if (Rows[i].Count != fieldCount) { throw new InvalidOperationException("Number of fields in each row do not match."); @@ -29,12 +31,16 @@ public UtfHeader GetHeader(out UtfFieldImage[] orderedDataFieldImages) { RowSize = GetSingleRowDataSize(), PerRowDataOffset = GetRowDescriptorSize() + SchemaOffset // "per row" data offset, actually }; + var perRowDataSize = header.RowCount * header.RowSize; + header.StringTableOffset = header.PerRowDataOffset + perRowDataSize; header.TableNameOffset = 0; var stringTableSize = GetStringTableSize(); + header.ExtraDataOffset = header.StringTableOffset + stringTableSize; + var extraDataSize = GetExtraDataSize(header, out orderedDataFieldImages); header.TableSize = header.ExtraDataOffset + extraDataSize; @@ -44,10 +50,13 @@ public UtfHeader GetHeader(out UtfFieldImage[] orderedDataFieldImages) { private void ProcessStringTable() { TableNameBytesCache = Encoding.ASCII.GetBytes(TableName).Append(0); + // Prepare static strings first. var row = Rows[0]; + foreach (var fieldImage in row) { fieldImage.NameBytesCache = Encoding.ASCII.GetBytes(fieldImage.Name).Append(0); + switch (fieldImage.Storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -59,16 +68,18 @@ private void ProcessStringTable() { } } break; - default: - break; } } + var row0 = row; + for (var i = 1; i < Rows.Count; ++i) { for (var j = 0; j < Rows[i].Count; ++j) { var fieldImage = Rows[i][j]; + fieldImage.Name = row0[j].Name; fieldImage.NameBytesCache = row0[j].NameBytesCache; + switch (fieldImage.Storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -77,11 +88,10 @@ private void ProcessStringTable() { fieldImage.StringValueBytesCache = row0[j].StringValueBytesCache; } break; - default: - break; } } } + // Per-row strings. foreach (var r in Rows) { foreach (var fieldImage in r) { @@ -111,11 +121,13 @@ private void SetStringFieldRelocations(UtfHeader header) { var currentStringTableOffset = (uint)TableNameBytesCache.Length; var row0 = Rows[0]; var row = row0; + // Field names and static strings foreach (var fieldImage in row) { fieldImage.NameOffset = currentStringTableOffset; currentStringTableOffset += (uint)fieldImage.NameBytesCache.Length; } + foreach (var fieldImage in row) { switch (fieldImage.Storage) { case ColumnStorage.Constant: @@ -125,14 +137,15 @@ private void SetStringFieldRelocations(UtfHeader header) { currentStringTableOffset += (uint)fieldImage.StringValueBytesCache.Length; } break; - default: - break; } } + for (var i = 1; i < Rows.Count; ++i) { for (var j = 0; j < Rows[i].Count; ++j) { var fieldImage = Rows[i][j]; + fieldImage.NameOffset = row0[j].NameOffset; + switch (fieldImage.Storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -140,11 +153,10 @@ private void SetStringFieldRelocations(UtfHeader header) { fieldImage.StringOffset = row0[j].StringOffset; } break; - default: - break; } } } + // Per-row strings foreach (var r in Rows) { foreach (var fieldImage in r) { @@ -167,16 +179,20 @@ private void SetExtraDataFieldRelocations(UtfHeader header, UtfFieldImage[] orde if (fieldImage.IsTable) { currentOffset = SerializationHelper.RoundUpAsTable(currentOffset, Alignment); } + fieldImage.DataOffset = currentOffset - baseOffset; currentOffset += (uint)fieldImage.DataValue.Length; } else { fieldImage.DataOffset = 0; } } + var row0 = Rows[0]; + for (var i = 1; i < Rows.Count; ++i) { for (var j = 0; j < Rows[i].Count; ++j) { var fieldImage = Rows[i][j]; + if (fieldImage.Type == ColumnType.Data && (fieldImage.Storage == ColumnStorage.Constant || fieldImage.Storage == ColumnStorage.Constant2)) { fieldImage.DataOffset = row0[j].DataOffset; } @@ -186,7 +202,7 @@ private void SetExtraDataFieldRelocations(UtfHeader header, UtfFieldImage[] orde private static readonly byte[] EmptyByteArray = new byte[0]; private static readonly byte[] EmptyAsciiStringBytes = new byte[1]; - private static readonly ushort SchemaOffset = 0x20; + private const ushort SchemaOffset = 0x20; } } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs index 282e4fe..b32d4b2 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; @@ -7,11 +7,14 @@ internal sealed partial class UtfTableImage { private uint GetRowDescriptorSize() { uint totalSize = 0; + foreach (var row in Rows) { foreach (var fieldImage in row) { // Storage descriptor (1 byte) + field name offset (4 bytes) totalSize += 5; + var storage = fieldImage.Storage; + switch (storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -56,6 +59,7 @@ private uint GetRowDescriptorSize() { } } } + return totalSize; } @@ -65,12 +69,16 @@ private uint GetRowDescriptorSize() { /// private ushort GetSingleRowDataSize() { ushort totalSize = 0; + var row = Rows[0]; + foreach (var fieldImage in row) { var storage = fieldImage.Storage; + switch (storage) { case ColumnStorage.PerRow: var type = fieldImage.Type; + switch (type) { case ColumnType.Byte: case ColumnType.SByte: @@ -111,15 +119,19 @@ private ushort GetSingleRowDataSize() { throw new ArgumentOutOfRangeException(nameof(storage)); } } + return totalSize; } private uint GetStringTableSize() { uint totalSize = 0; + totalSize += (uint)TableNameBytesCache.Length; totalSize += Rows[0].Aggregate(0, (current, fieldImage) => current + (uint)fieldImage.NameBytesCache.Length); + for (var i = 0; i < Rows.Count; ++i) { var row = Rows[i]; + foreach (var fieldImage in row) { switch (fieldImage.Type) { case ColumnType.String: @@ -133,12 +145,8 @@ private uint GetStringTableSize() { case ColumnStorage.PerRow: totalSize += (uint)fieldImage.StringValueBytesCache.Length; break; - default: - break; } break; - default: - break; } } } @@ -152,11 +160,13 @@ private uint GetExtraDataSize(UtfHeader header, out UtfFieldImage[] orderedDataF var fieldImagesInNewOrder = new List(); var row0 = Rows[0]; var firstTableDataFieldIndex = -1; + foreach (var fieldImage in row0) { if (fieldImage.Type == ColumnType.Data && (fieldImage.Storage == ColumnStorage.Constant || fieldImage.Storage == ColumnStorage.Constant2)) { AddToNewOrdererdDataFieldList(fieldImagesInNewOrder, fieldImage, ref firstTableDataFieldIndex); } } + foreach (var row in Rows) { foreach (var fieldImage in row) { if (fieldImage.Type == ColumnType.Data && fieldImage.Storage == ColumnStorage.PerRow) { @@ -170,16 +180,20 @@ private uint GetExtraDataSize(UtfHeader header, out UtfFieldImage[] orderedDataF baseOffset = SerializationHelper.RoundUpAsTable(baseOffset, Alignment); header.ExtraDataOffset = baseOffset; } + orderedDataFieldImages = fieldImagesInNewOrder.ToArray(); foreach (var fieldImage in orderedDataFieldImages) { var rawOffset = baseOffset; + if (fieldImage.IsTable && fieldImage.DataValue.Length > 0) { baseOffset = SerializationHelper.RoundUpAsTable(baseOffset, Alignment); } + baseOffset += (uint)fieldImage.DataValue.Length; totalSize += (baseOffset - rawOffset); } + return totalSize; } From 402c7f4e1d67da0e35ebf0a338f755b305c81b4c Mon Sep 17 00:00:00 2001 From: George Wu Date: Mon, 26 Feb 2018 01:45:22 +0100 Subject: [PATCH 3/4] MLTD-compatible ACB generation. --- .../Serialization/SerializationHelper.cs | 6 +- .../Serialization/UtfFieldImage.cs | 11 +- .../UtfTableImage.Preprocessor.cs | 21 ++- .../UtfTableImage.SizeCalculation.cs | 133 ++++++++++-------- 4 files changed, 100 insertions(+), 71 deletions(-) diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs index 1eaf5b2..d59a8fb 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/SerializationHelper.cs @@ -12,9 +12,9 @@ public static uint RoundUpAsTable(uint value, uint alignment) { // This action seems weird. But it does exist (see Cue table in CGSS song_1001[oneshin]), I don't know why. value = AcbHelper.RoundUpToAlignment(value, 4); - //if (value % alignment == 0) { - // value += alignment; - //} + if (value % alignment == 0) { + value += alignment; + } return AcbHelper.RoundUpToAlignment(value, alignment); } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs index 789de8c..1205afe 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfFieldImage.cs @@ -26,6 +26,7 @@ internal sealed class UtfFieldImage : IComparable { public void SetValue(object value) { ColumnType type; var union = new NumericUnion(); + if (value is byte) { type = ColumnType.Byte; union.U8 = (byte)value; @@ -65,7 +66,9 @@ public void SetValue(object value) { } else { throw new ArgumentException("Unsupported argument type.", nameof(value)); } + Type = type; + switch (type) { case ColumnType.String: case ColumnType.Data: @@ -128,7 +131,13 @@ public void WriteValueTo(Stream stream) { break; case ColumnType.Data: stream.WriteUInt32BE(DataOffset); - stream.WriteUInt32BE((uint)DataValue.Length); + + // Tables ending locations, 4-byte alignment. + if (IsTable) { + stream.WriteUInt32BE(AcbHelper.RoundUpToAlignment((uint)DataValue.Length, 4)); + } else { + stream.WriteUInt32BE((uint)DataValue.Length); + } break; default: throw new ArgumentOutOfRangeException(nameof(Type)); diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs index 78048a4..6e82a5f 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.Preprocessor.cs @@ -23,13 +23,16 @@ public UtfHeader GetHeader(out UtfFieldImage[] orderedDataFieldImages) { } } + var singleRowDataSize = GetSingleRowDataSize(); + var rowDescriptorSize = GetRowDescriptorSize(); + var header = new UtfHeader { TableName = TableName, RowCount = (uint)Rows.Count, FieldCount = (ushort)fieldCount, Unknown1 = 1, - RowSize = GetSingleRowDataSize(), - PerRowDataOffset = GetRowDescriptorSize() + SchemaOffset // "per row" data offset, actually + RowSize = singleRowDataSize, + PerRowDataOffset = rowDescriptorSize + SchemaOffset // "per row" data offset, actually }; var perRowDataSize = header.RowCount * header.RowSize; @@ -45,6 +48,9 @@ public UtfHeader GetHeader(out UtfFieldImage[] orderedDataFieldImages) { header.TableSize = header.ExtraDataOffset + extraDataSize; + // ??? + header.TableSize = AcbHelper.RoundUpToAlignment(header.TableSize, 4); + return header; } @@ -120,15 +126,14 @@ private void SetStringFieldRelocations(UtfHeader header) { // String table var currentStringTableOffset = (uint)TableNameBytesCache.Length; var row0 = Rows[0]; - var row = row0; // Field names and static strings - foreach (var fieldImage in row) { + foreach (var fieldImage in row0) { fieldImage.NameOffset = currentStringTableOffset; currentStringTableOffset += (uint)fieldImage.NameBytesCache.Length; } - foreach (var fieldImage in row) { + foreach (var fieldImage in row0) { switch (fieldImage.Storage) { case ColumnStorage.Constant: case ColumnStorage.Constant2: @@ -181,7 +186,13 @@ private void SetExtraDataFieldRelocations(UtfHeader header, UtfFieldImage[] orde } fieldImage.DataOffset = currentOffset - baseOffset; + currentOffset += (uint)fieldImage.DataValue.Length; + + // Extra 4-byte alignment at the end of tables. + if (fieldImage.IsTable) { + currentOffset = AcbHelper.RoundUpToAlignment(currentOffset, Alignment); + } } else { fieldImage.DataOffset = 0; } diff --git a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs index b32d4b2..6196c93 100644 --- a/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs +++ b/Exchange/DereTore.Exchange.Archive.ACB/Serialization/UtfTableImage.SizeCalculation.cs @@ -8,55 +8,54 @@ internal sealed partial class UtfTableImage { private uint GetRowDescriptorSize() { uint totalSize = 0; - foreach (var row in Rows) { - foreach (var fieldImage in row) { - // Storage descriptor (1 byte) + field name offset (4 bytes) - totalSize += 5; - - var storage = fieldImage.Storage; - - switch (storage) { - case ColumnStorage.Constant: - case ColumnStorage.Constant2: - var type = fieldImage.Type; - switch (type) { - case ColumnType.Byte: - case ColumnType.SByte: - totalSize += 1; - break; - case ColumnType.UInt16: - case ColumnType.Int16: - totalSize += 2; - break; - case ColumnType.UInt32: - case ColumnType.Int32: - totalSize += 4; - break; - case ColumnType.UInt64: - case ColumnType.Int64: - totalSize += 8; - break; - case ColumnType.Single: - totalSize += 4; - break; - case ColumnType.Double: - totalSize += 8; - break; - case ColumnType.String: - totalSize += 4; - break; - case ColumnType.Data: - totalSize += 4 + 4; - break; - default: - throw new ArgumentOutOfRangeException(nameof(type)); - } - break; - case ColumnStorage.PerRow: - break; - default: - throw new ArgumentOutOfRangeException(nameof(storage)); - } + foreach (var fieldImage in Rows[0]) { + // Storage descriptor (1 byte) + field name offset (4 bytes) + totalSize += 5; + + var storage = fieldImage.Storage; + + switch (storage) { + case ColumnStorage.Constant: + case ColumnStorage.Constant2: + var type = fieldImage.Type; + + switch (type) { + case ColumnType.Byte: + case ColumnType.SByte: + totalSize += 1; + break; + case ColumnType.UInt16: + case ColumnType.Int16: + totalSize += 2; + break; + case ColumnType.UInt32: + case ColumnType.Int32: + totalSize += 4; + break; + case ColumnType.UInt64: + case ColumnType.Int64: + totalSize += 8; + break; + case ColumnType.Single: + totalSize += 4; + break; + case ColumnType.Double: + totalSize += 8; + break; + case ColumnType.String: + totalSize += 4; + break; + case ColumnType.Data: + totalSize += 4 + 4; + break; + default: + throw new ArgumentOutOfRangeException(nameof(type)); + } + break; + case ColumnStorage.PerRow: + break; + default: + throw new ArgumentOutOfRangeException(nameof(storage)); } } @@ -186,11 +185,18 @@ private uint GetExtraDataSize(UtfHeader header, out UtfFieldImage[] orderedDataF foreach (var fieldImage in orderedDataFieldImages) { var rawOffset = baseOffset; + // Tables' starting locations are rounded by Alignment (usually 0x20). if (fieldImage.IsTable && fieldImage.DataValue.Length > 0) { baseOffset = SerializationHelper.RoundUpAsTable(baseOffset, Alignment); } baseOffset += (uint)fieldImage.DataValue.Length; + + // Tables' ending locations are rounded by 4. + if (fieldImage.IsTable && fieldImage.DataValue.Length > 0) { + baseOffset = AcbHelper.RoundUpToAlignment(baseOffset, 4); + } + totalSize += (baseOffset - rawOffset); } @@ -200,19 +206,22 @@ private uint GetExtraDataSize(UtfHeader header, out UtfFieldImage[] orderedDataF private static void AddToNewOrdererdDataFieldList(List fieldImagesInNewOrder, UtfFieldImage fieldImage, ref int firstTableDataFieldIndex) { // Tables are always the last fields to store. // Priority: [static misc data] > [dynamic static data] > [static table] > [dynamic table] - if (fieldImage.IsTable) { - fieldImagesInNewOrder.Add(fieldImage); - if (firstTableDataFieldIndex < 0) { - firstTableDataFieldIndex = fieldImagesInNewOrder.Count - 1; - } - } else { - if (firstTableDataFieldIndex < 0) { - fieldImagesInNewOrder.Add(fieldImage); - } else { - fieldImagesInNewOrder.Insert(firstTableDataFieldIndex, fieldImage); - ++firstTableDataFieldIndex; - } - } + //if (fieldImage.IsTable) { + // fieldImagesInNewOrder.Add(fieldImage); + // if (firstTableDataFieldIndex < 0) { + // firstTableDataFieldIndex = fieldImagesInNewOrder.Count - 1; + // } + //} else { + // if (firstTableDataFieldIndex < 0) { + // fieldImagesInNewOrder.Add(fieldImage); + // } else { + // fieldImagesInNewOrder.Insert(firstTableDataFieldIndex, fieldImage); + // ++firstTableDataFieldIndex; + // } + //} + + // No sorting. + fieldImagesInNewOrder.Add(fieldImage); } } From 5974a6cf6d15f58ddb5ccdf455493c9434958217 Mon Sep 17 00:00:00 2001 From: George Wu Date: Mon, 26 Feb 2018 02:09:23 +0100 Subject: [PATCH 4/4] [hcaenc] Added quality support. --- Apps/Hcaenc/DereTore.Apps.Hcaenc.csproj | 5 +++++ Apps/Hcaenc/Options.cs | 16 ++++++++++++++ Apps/Hcaenc/Program.cs | 28 ++++++++++++++++++------- Apps/Hcaenc/packages.config | 4 ++++ 4 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 Apps/Hcaenc/Options.cs create mode 100644 Apps/Hcaenc/packages.config diff --git a/Apps/Hcaenc/DereTore.Apps.Hcaenc.csproj b/Apps/Hcaenc/DereTore.Apps.Hcaenc.csproj index 36a5edc..8a87cec 100644 --- a/Apps/Hcaenc/DereTore.Apps.Hcaenc.csproj +++ b/Apps/Hcaenc/DereTore.Apps.Hcaenc.csproj @@ -54,14 +54,19 @@ 6 + + ..\..\packages\CommandLineParser.2.2.1\lib\net40\CommandLine.dll + + +