From 90b5d5f26911f173028eb1b91c6c7ca9dc751269 Mon Sep 17 00:00:00 2001 From: morkt Date: Mon, 22 Jan 2018 11:41:46 +0400 Subject: [PATCH] (NonColor): reworked archive index reading. --- ArcFormats/ArcFormats.csproj | 1 + ArcFormats/NonColor/ArcACV.cs | 16 +-- ArcFormats/NonColor/ArcDAT.cs | 215 ++++++++++++++++++++++--------- ArcFormats/NonColor/ArcMinato.cs | 102 +++++++++++++++ 4 files changed, 262 insertions(+), 72 deletions(-) create mode 100644 ArcFormats/NonColor/ArcMinato.cs diff --git a/ArcFormats/ArcFormats.csproj b/ArcFormats/ArcFormats.csproj index 16cb72be..1097d01b 100644 --- a/ArcFormats/ArcFormats.csproj +++ b/ArcFormats/ArcFormats.csproj @@ -136,6 +136,7 @@ + diff --git a/ArcFormats/NonColor/ArcACV.cs b/ArcFormats/NonColor/ArcACV.cs index a782ebc3..2f417281 100644 --- a/ArcFormats/NonColor/ArcACV.cs +++ b/ArcFormats/NonColor/ArcACV.cs @@ -59,19 +59,19 @@ namespace GameRes.Formats.NonColor bool is_script = VFS.IsPathEqualsToFileName (file.Name, "script.dat"); - var dir = new List (count); - using (var input = file.CreateStream (8, (uint)count * 0x15)) + using (var index = new NcIndexReader (file, count, key) { IndexPosition = 8 }) { - foreach (var entry in ReadIndex (input, count, scheme, key)) + var file_map = ReadFilenameMap (scheme); + var dir = index.Read (file_map); + if (null == dir) + return null; + if (is_script) { - if (!entry.CheckPlacement (file.MaxOffset)) - return null; - if (is_script) + foreach (ArcDatEntry entry in dir) entry.Hash ^= scheme.Hash; - dir.Add (entry); } + return new ArcFile (file, this, dir); } - return new ArcFile (file, this, dir); } public override Stream OpenEntry (ArcFile arc, Entry entry) diff --git a/ArcFormats/NonColor/ArcDAT.cs b/ArcFormats/NonColor/ArcDAT.cs index 49540120..6a40b343 100644 --- a/ArcFormats/NonColor/ArcDAT.cs +++ b/ArcFormats/NonColor/ArcDAT.cs @@ -26,6 +26,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Composition; +using System.Diagnostics; using System.IO; using GameRes.Compression; using GameRes.Utility; @@ -56,12 +57,18 @@ namespace GameRes.Formats.NonColor { public string Title; public ulong Hash; + public string FileListName; public Scheme(string title) { Title = title; var key = Encodings.cp932.GetBytes(title); - Hash = Crc64.Compute(key, 0, key.Length); + Hash = ComputeHash (key); + } + + public virtual ulong ComputeHash (byte[] name) + { + return Crc64.Compute (name, 0, name.Length); } } @@ -69,12 +76,6 @@ namespace GameRes.Formats.NonColor public class ArcDatScheme : ResourceScheme { public Dictionary KnownSchemes; - - public static ulong GetKey (string title) - { - var key = Encodings.cp932.GetBytes (title); - return Crc64.Compute (key, 0, key.Length); - } } public class ArcDatOptions : ResourceOptions @@ -82,6 +83,12 @@ namespace GameRes.Formats.NonColor public string Scheme; } + internal struct NameRecord + { + public string Name; + public byte[] NameBytes; + } + [Export(typeof(ArchiveFormat))] public class DatOpener : ArchiveFormat { @@ -96,13 +103,15 @@ namespace GameRes.Formats.NonColor Extensions = new string[] { "dat" }; } + internal const int SignatureKey = 0x26ACA46E; + public static readonly string PersistentFileMapName = "NCFileMap.dat"; public override ArcFile TryOpen (ArcView file) { if (!file.Name.HasExtension (".dat")) return null; - int count = file.View.ReadInt32 (0) ^ 0x26ACA46E; + int count = file.View.ReadInt32 (0) ^ SignatureKey; if (!IsSaneCount (count)) return null; @@ -110,17 +119,14 @@ namespace GameRes.Formats.NonColor if (null == scheme) return null; - var dir = new List (count); - using (var input = file.CreateStream (4, (uint)count * 0x15)) + using (var index = new NcIndexReader (file, count)) { - foreach (var entry in ReadIndex (input, count, scheme)) - { - if (!entry.CheckPlacement (file.MaxOffset)) - return null; - dir.Add (entry); - } + var file_map = ReadFilenameMap (scheme); + var dir = index.Read (file_map); + if (null == dir) + return null; + return new ArcDatArchive (file, this, dir, scheme.Hash); } - return new ArcDatArchive (file, this, dir, scheme.Hash); } public override Stream OpenEntry (ArcFile arc, Entry entry) @@ -142,49 +148,6 @@ namespace GameRes.Formats.NonColor return new BinMemoryStream (data, entry.Name); } - internal IEnumerable ReadIndex (IBinaryStream input, int count, Scheme scheme, uint key = 0) - { - int skipped = 0; - var file_map = ReadFilenameMap (scheme); - for (int i = 0; i < count; ++i) - { - var hash = input.ReadUInt64(); - var entry = new ArcDatEntry - { - Hash = hash, - Flags = input.ReadByte() ^ (byte)hash, - Offset = input.ReadUInt32() ^ (uint)hash ^ key, - Size = input.ReadUInt32() ^ (uint)hash, - UnpackedSize = input.ReadUInt32() ^ (uint)hash, - }; - entry.IsPacked = 0 != (entry.Flags & 2); - byte[] raw_name = null; - if (file_map.TryGetValue (hash, out raw_name)) - { - entry.Name = Encodings.cp932.GetString (raw_name); - entry.Type = FormatCatalog.Instance.GetTypeFromName (entry.Name); - entry.RawName = raw_name; - } - if (0 == (entry.Flags & 2)) - { - if (null == raw_name) - { -// System.Diagnostics.Trace.WriteLine ("Unknown hash", hash.ToString ("X16")); - ++skipped; - continue; - } - entry.Offset ^= raw_name[raw_name.Length >> 1]; - entry.Size ^= raw_name[raw_name.Length >> 2]; - entry.UnpackedSize ^= raw_name[raw_name.Length >> 3]; - } - if (string.IsNullOrEmpty (entry.Name)) - entry.Name = hash.ToString ("X16"); - yield return entry; - } - if (skipped != 0) - System.Diagnostics.Trace.WriteLine (string.Format ("Missing {0} names", skipped), "[noncolor]"); - } - internal unsafe void DecryptData (byte[] data, uint key) { fixed (byte* data8 = data) @@ -221,12 +184,23 @@ namespace GameRes.Formats.NonColor return map; } - Tuple> LastAccessedScheme; + Tuple> LastAccessedScheme; - internal IDictionary ReadFilenameMap (Scheme scheme) + internal IDictionary ReadFilenameMap (Scheme scheme) { if (null != LastAccessedScheme && LastAccessedScheme.Item1 == scheme.Hash) return LastAccessedScheme.Item2; + if (!string.IsNullOrEmpty (scheme.FileListName)) + { + var dict = new Dictionary(); + FormatCatalog.Instance.ReadFileList (scheme.FileListName, line => { + var bytes = line.ToLowerShiftJis(); + ulong hash = scheme.ComputeHash (bytes); + dict[hash] = new NameRecord { Name = line, NameBytes = bytes }; + }); + LastAccessedScheme = Tuple.Create (scheme.Hash, dict); + return dict; + } var dir = FormatCatalog.Instance.DataDirectory; var lst_file = Path.Combine (dir, PersistentFileMapName); var idx_file = Path.ChangeExtension (lst_file, ".idx"); @@ -243,7 +217,7 @@ namespace GameRes.Formats.NonColor using (var lst_stream = File.OpenRead (lst_file)) using (var lst = new BinaryReader (lst_stream)) { - var name_map = new Dictionary (nc_info.Item2); + var name_map = new Dictionary (nc_info.Item2); idx_stream.Position = nc_info.Item1; for (int i = 0; i < nc_info.Item2; ++i) { @@ -251,7 +225,9 @@ namespace GameRes.Formats.NonColor uint offset = idx.ReadUInt32(); int length = idx.ReadInt32(); lst_stream.Position = offset; - name_map[key] = lst.ReadBytes (length); + var name_bytes = lst.ReadBytes (length); + var name = Encodings.cp932.GetString (name_bytes); + name_map[key] = new NameRecord { Name = name, NameBytes = name_bytes }; } LastAccessedScheme = Tuple.Create (scheme.Hash, name_map); return name_map; @@ -289,4 +265,115 @@ namespace GameRes.Formats.NonColor return new GUI.WidgetNCARC(); } } + + internal abstract class NcIndexReaderBase : IDisposable + { + protected IBinaryStream m_input; + private List m_dir; + private int m_count; + private long m_max_offset; + + public long IndexPosition { get; set; } + public long MaxOffset { get { return m_max_offset; } } + + protected NcIndexReaderBase (ArcView file, int count) + { + m_input = file.CreateStream(); + m_dir = new List (count); + m_count = count; + m_max_offset = file.MaxOffset; + IndexPosition = 4; + } + + public List Read (IDictionary file_map) + { + int skipped = 0; + string last_name = null; + m_input.Position = IndexPosition; + for (int i = 0; i < m_count; ++i) + { + var entry = ReadEntry(); + NameRecord known_rec; + if (file_map.TryGetValue (entry.Hash, out known_rec)) + { + entry.Name = known_rec.Name; + entry.Type = FormatCatalog.Instance.GetTypeFromName (entry.Name); + entry.RawName = known_rec.NameBytes; + } + if (0 == (entry.Flags & 2)) + { + if (null == known_rec.Name) + { + if (last_name != null) + Trace.WriteLine (string.Format ("[{0}] {1}", i-1, last_name), "[noncolor]"); + Trace.WriteLine (string.Format ("[{0}] Unknown hash {1:X8}", i, entry.Hash), "[noncolor]"); + last_name = null; + ++skipped; + continue; + } + else + { + var raw_name = known_rec.NameBytes; + if (null == last_name && i > 0) + Trace.WriteLine (string.Format ("[{0}] {1}", i, known_rec.Name), "[noncolor]"); + entry.Offset ^= raw_name[raw_name.Length >> 1]; + entry.Size ^= raw_name[raw_name.Length >> 2]; + entry.UnpackedSize ^= raw_name[raw_name.Length >> 3]; + } + } + last_name = known_rec.Name; + if (!entry.CheckPlacement (MaxOffset)) + return null; + if (string.IsNullOrEmpty (entry.Name)) + entry.Name = string.Format ("{0:D5}#{1:X8}", i, entry.Hash); + m_dir.Add (entry); + } + if (skipped != 0) + Trace.WriteLine (string.Format ("Missing {0} names", skipped), "[noncolor]"); + return m_dir; + } + + protected abstract ArcDatEntry ReadEntry (); + + #region IDisposable Members + bool m_disposed = false; + public void Dispose () + { + Dispose (true); + GC.SuppressFinalize (this); + } + + protected virtual void Dispose (bool disposing) + { + if (!m_disposed) + { + m_input.Dispose(); + m_disposed = true; + } + } + #endregion + } + + internal class NcIndexReader : NcIndexReaderBase + { + readonly uint m_master_key; + + public NcIndexReader (ArcView file, int count, uint master_key = 0) : base (file, count) + { + m_master_key = master_key; + } + + protected override ArcDatEntry ReadEntry () + { + var hash = m_input.ReadUInt64(); + int flags = m_input.ReadByte() ^ (byte)hash; + return new ArcDatEntry { + Hash = hash, + Offset = m_input.ReadUInt32() ^ (uint)hash ^ m_master_key, + Size = m_input.ReadUInt32() ^ (uint)hash, + UnpackedSize = m_input.ReadUInt32() ^ (uint)hash, + IsPacked = 0 != (flags & 2), + }; + } + } } diff --git a/ArcFormats/NonColor/ArcMinato.cs b/ArcFormats/NonColor/ArcMinato.cs new file mode 100644 index 00000000..3402d047 --- /dev/null +++ b/ArcFormats/NonColor/ArcMinato.cs @@ -0,0 +1,102 @@ +//! \file ArcDAT.cs +//! \date 2018 Jan 17 +//! \brief MinatoSoft resource archive. +// +// Copyright (C) 2018 by morkt +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using GameRes.Utility; +using GameRes.Formats.NonColor; + +namespace GameRes.Formats.Minato +{ + [Serializable] + public class NcSchemeCrc32 : Scheme + { + public NcSchemeCrc32 (string title) : base (title) { } + + public override ulong ComputeHash (byte[] name) + { + return Crc32.Compute (name, 0, name.Length); + } + } + + /// + /// This format is mostly identical to NonColor.DatOpener, but uses CRC32 for hashes and big-endian + /// byte order. + /// + [Export(typeof(ArchiveFormat))] + public class MinatoDatOpener : NonColor.DatOpener + { + public override string Tag { get { return "DAT/MINATO"; } } + public override string Description { get { return "MinatoSoft resource archive"; } } + public override uint Signature { get { return 0; } } + public override bool IsHierarchic { get { return true; } } + public override bool CanWrite { get { return false; } } + + public override ArcFile TryOpen (ArcView file) + { + if (!file.Name.HasExtension (".dat")) + return null; + int count = Binary.BigEndian (file.View.ReadInt32 (0)) ^ SignatureKey; + if (!IsSaneCount (count)) + return null; + + var scheme = QueryScheme (file.Name); + if (null == scheme) + return null; + + using (var index = new MinatoIndexReader (file, count)) + { + var file_map = ReadFilenameMap (scheme); + var dir = index.Read (file_map); + if (null == dir) + return null; + return new ArcDatArchive (file, this, dir, scheme.Hash); + } + } + } + + internal class MinatoIndexReader : NcIndexReaderBase + { + public MinatoIndexReader (ArcView file, int count) : base (file, count) { } + + protected override ArcDatEntry ReadEntry () + { + uint key = Binary.BigEndian (m_input.ReadUInt32()); + int flags = m_input.ReadUInt8() ^ (byte)key; + uint offset = Binary.BigEndian (m_input.ReadUInt32()) ^ key; + uint packed_size = Binary.BigEndian (m_input.ReadUInt32()) ^ key; + uint unpacked_size = Binary.BigEndian (m_input.ReadUInt32()) ^ key; + return new ArcDatEntry { + Hash = key, + Flags = flags, + Offset = offset, + Size = packed_size, + UnpackedSize = unpacked_size, + IsPacked = 0 != (flags & 2), + }; + } + } +}