From 2d6dec4dfc869ef6bbfcf97483e2a089e02c8e9d Mon Sep 17 00:00:00 2001 From: morkt Date: Sun, 12 Mar 2017 05:45:36 +0400 Subject: [PATCH] (QLIE): implemented FilePackVer3.1 --- ArcFormats/ArcFormats.csproj | 1 + ArcFormats/Qlie/ArcQLIE.cs | 222 +++++----------------- ArcFormats/Qlie/Encryption.cs | 339 ++++++++++++++++++++++++++++++++++ 3 files changed, 383 insertions(+), 179 deletions(-) create mode 100644 ArcFormats/Qlie/Encryption.cs diff --git a/ArcFormats/ArcFormats.csproj b/ArcFormats/ArcFormats.csproj index 1935346e..bfce508c 100644 --- a/ArcFormats/ArcFormats.csproj +++ b/ArcFormats/ArcFormats.csproj @@ -104,6 +104,7 @@ + diff --git a/ArcFormats/Qlie/ArcQLIE.cs b/ArcFormats/Qlie/ArcQLIE.cs index d6e9c028..acb95fd9 100644 --- a/ArcFormats/Qlie/ArcQLIE.cs +++ b/ArcFormats/Qlie/ArcQLIE.cs @@ -51,23 +51,12 @@ namespace GameRes.Formats.Qlie internal class QlieArchive : ArcFile { - /// - /// Hash generated from the key data contained within archive index. - /// - public uint Hash; + public readonly IEncryption Encryption; - /// - /// Internal game data used to decrypt encrypted entries. - /// null if not used. - /// - public byte[] GameKeyData; - - public QlieArchive (ArcView arc, ArchiveFormat impl, ICollection dir, - uint hash, byte[] key_data) + public QlieArchive (ArcView arc, ArchiveFormat impl, ICollection dir, IEncryption enc) : base (arc, impl, dir) { - Hash = hash; - GameKeyData = key_data; + Encryption = enc; } } @@ -115,11 +104,10 @@ namespace GameRes.Formats.Qlie return null; long index_offset = file.MaxOffset - 0x1c; if (!file.View.AsciiEqual (index_offset, "FilePackVer") - || !file.View.AsciiEqual (index_offset+0xC, ".0")) + || '.' != file.View.ReadByte (index_offset+0xC)) return null; - int pack_version = file.View.ReadByte (index_offset+0xB) - '0'; - if (pack_version < 1 || pack_version > 3) - throw new NotSupportedException ("Not supported QLIE archive version"); + var pack_version = new Version (file.View.ReadByte (index_offset+0xB) - '0', + file.View.ReadByte (index_offset+0xD) - '0'); int count = file.View.ReadInt32 (index_offset+0x10); if (!IsSaneCount (count)) return null; @@ -129,9 +117,8 @@ namespace GameRes.Formats.Qlie byte[] arc_key = null; byte[] key_file = null; - uint name_key = 0xC4; // default name encryption key for versions 1 and 2 bool use_pack_keyfile = false; - if (3 == pack_version) + if (pack_version.Major >= 3) { key_file = FindKeyFile (file); use_pack_keyfile = key_file != null; @@ -139,51 +126,45 @@ namespace GameRes.Formats.Qlie if (use_pack_keyfile) arc_key = QueryEncryption (file); // use_pack_keyfile = null != arc_key; - - var key_data = file.View.ReadBytes (file.MaxOffset-0x41C, 0x100); - name_key = GenerateKey (key_data) & 0x0FFFFFFFu; } + var enc = QlieEncryption.Create (file, pack_version, arc_key); var name_buffer = new byte[0x100]; var dir = new List (count); - for (int i = 0; i < count; ++i) + using (var index = file.CreateStream (index_offset)) { - int name_length = file.View.ReadUInt16 (index_offset); - if (name_length > name_buffer.Length) - name_buffer = new byte[name_length]; - if (name_length != file.View.Read (index_offset+2, name_buffer, 0, (uint)name_length)) - return null; - - int key = name_length + ((int)name_key ^ 0x3e); - for (int k = 0; k < name_length; ++k) - name_buffer[k] ^= (byte)(((k + 1) ^ key) + k + 1); - - string name = Encodings.cp932.GetString (name_buffer, 0, name_length); - var entry = FormatCatalog.Instance.Create (name); - if (use_pack_keyfile) - entry.RawName = name_buffer.Take (name_length).ToArray(); - - index_offset += 2 + name_length; - entry.Offset = file.View.ReadInt64 (index_offset); - entry.Size = file.View.ReadUInt32 (index_offset+8); - if (!entry.CheckPlacement (file.MaxOffset)) - return null; - entry.UnpackedSize = file.View.ReadUInt32 (index_offset+12); - entry.IsPacked = 0 != file.View.ReadInt32 (index_offset+0x10); - entry.IsEncrypted = 0 != file.View.ReadInt32 (index_offset+0x14); - entry.Hash = file.View.ReadUInt32 (index_offset+0x18); - entry.KeyFile = key_file; - if (3 == pack_version && use_pack_keyfile && entry.Name.Contains ("pack_keyfile")) + for (int i = 0; i < count; ++i) { - // note that 'pack_keyfile' itself is encrypted using 'key.fkey' file contents. - key_file = ReadEntryBytes (file, entry, name_key, arc_key); + int name_length = index.ReadUInt16(); + if (enc.IsUnicode) + name_length *= 2; + if (name_length > name_buffer.Length) + name_buffer = new byte[name_length]; + if (name_length != index.Read (name_buffer, 0, name_length)) + return null; + var name = enc.DecryptName (name_buffer, name_length); + var entry = FormatCatalog.Instance.Create (name); + if (use_pack_keyfile) + entry.RawName = name_buffer.Take (name_length).ToArray(); + + entry.Offset = index.ReadInt64(); // [+00] + entry.Size = index.ReadUInt32(); // [+08] + if (!entry.CheckPlacement (file.MaxOffset)) + return null; + entry.UnpackedSize = index.ReadUInt32(); // [+0C] + entry.IsPacked = 0 != index.ReadInt32(); // [+10] + entry.IsEncrypted = 0 != index.ReadInt32(); // [+14] + entry.Hash = index.ReadUInt32(); // [+18] + entry.KeyFile = key_file; + if (3 == pack_version.Major && use_pack_keyfile && entry.Name.Contains ("pack_keyfile")) + { + // note that 'pack_keyfile' itself is encrypted using 'key.fkey' file contents. + key_file = ReadEntryBytes (file, entry, enc); + } + dir.Add (entry); } - dir.Add (entry); - index_offset += 0x1c; } - if (pack_version < 3) - name_key = 0; - return new QlieArchive (file, this, dir, name_key, arc_key); + return new QlieArchive (file, this, dir, enc); } public override Stream OpenEntry (ArcFile arc, Entry entry) @@ -192,125 +173,27 @@ namespace GameRes.Formats.Qlie var qarc = arc as QlieArchive; if (null == qent || null == qarc || (!qent.IsEncrypted && !qent.IsPacked)) return arc.File.CreateStream (entry.Offset, entry.Size); - var data = ReadEntryBytes (arc.File, qent, qarc.Hash, qarc.GameKeyData); + var data = ReadEntryBytes (arc.File, qent, qarc.Encryption); return new BinMemoryStream (data, entry.Name); } - private byte[] ReadEntryBytes (ArcView file, QlieEntry entry, uint hash, byte[] game_key) + private byte[] ReadEntryBytes (ArcView file, QlieEntry entry, IEncryption enc) { var data = file.View.ReadBytes (entry.Offset, entry.Size); if (entry.IsEncrypted) { - if (entry.KeyFile != null) - DecryptV3 (data, 0, data.Length, entry.RawName, hash, entry.KeyFile, game_key); - else - Decrypt (data, 0, data.Length, hash); + enc.DecryptEntry (data, 0, data.Length, entry); } if (entry.IsPacked) { - var unpacked = Decompress (data); - if (null != unpacked) - data = unpacked; + data = Decompress (data) ?? data; } return data; } - private void Decrypt (byte[] buffer, int offset, int length, uint key) - { - if (offset < 0) - throw new ArgumentOutOfRangeException ("offset"); - if (length > buffer.Length || offset > buffer.Length - length) - throw new ArgumentOutOfRangeException ("length"); - - ulong hash = 0xA73C5F9DA73C5F9Dul; - ulong xor = ((uint)length + key) ^ 0xFEC9753Eu; - xor |= xor << 32; - unsafe - { - fixed (byte* raw = buffer) - { - ulong* encoded = (ulong*)(raw + offset); - for (int i = 0; i < length / 8; ++i) - { - hash = MMX.PAddD (hash, 0xCE24F523CE24F523ul) ^ xor; - xor = *encoded ^ hash; - *encoded++ = xor; - } - } - } - } - - private void DecryptV3 (byte[] data, int offset, int length, byte[] file_name, - uint arc_hash, byte[] key_file, byte[] game_key) - { - // play it safe with 'unsafe' sections - if (offset < 0) - throw new ArgumentOutOfRangeException ("offset"); - if (length > data.Length || offset > data.Length - length) - throw new ArgumentOutOfRangeException ("length"); - - if (length < 8) - return; - - uint hash = 0x85F532; - uint seed = 0x33F641; - - for (uint i = 0; i < file_name.Length; i++) - { - hash += (i & 0xFF) * file_name[i]; - seed ^= hash; - } - - seed += arc_hash ^ (7 * ((uint)data.Length & 0xFFFFFF) + (uint)data.Length - + hash + (hash ^ (uint)data.Length ^ 0x8F32DCu)); - seed = 9 * (seed & 0xFFFFFF); - - if (game_key != null) - seed ^= 0x453A; - - var mt = new QlieMersenneTwister (seed); - if (key_file != null) - mt.XorState (key_file); - if (game_key != null) - mt.XorState (game_key); - - // game code fills dword[41] table, but only the first 16 qwords are used - ulong[] table = new ulong[16]; - for (int i = 0; i < table.Length; ++i) - table[i] = mt.Rand64(); - - // compensate for 9 discarded dwords - for (int i = 0; i < 9; ++i) - mt.Rand(); - - ulong hash64 = mt.Rand64(); - uint t = mt.Rand() & 0xF; - unsafe - { - fixed (byte* raw_data = &data[offset]) - { - ulong* data64 = (ulong*)raw_data; - int qword_length = length / 8; - for (int i = 0; i < qword_length; ++i) - { - hash64 = MMX.PAddD (hash64 ^ table[t], table[t]); - - ulong d = data64[i] ^ hash64; - data64[i] = d; - - hash64 = MMX.PAddB (hash64, d) ^ d; - hash64 = MMX.PAddW (MMX.PSllD (hash64, 1), d); - - t++; - t &= 0xF; - } - } - } - } - internal static byte[] Decompress (byte[] input) { - if (LittleEndian.ToUInt32 (input, 0) != 0xFF435031) + if (LittleEndian.ToUInt32 (input, 0) != 0xFF435031) // '1PC\xFF' return null; bool is_16bit = 0 != (input[4] & 1); @@ -393,25 +276,6 @@ namespace GameRes.Formats.Qlie return output; } - uint GenerateKey (byte[] key_data) - { - ulong hash = 0; - ulong key = 0; - unsafe - { - fixed (byte* data = key_data) - { - ulong* data64 = (ulong*)data; - for (int i = key_data.Length / 8; i > 0; --i) - { - hash = MMX.PAddW (hash, 0x0307030703070307); - key = MMX.PAddW (key, *data64++ ^ hash); - } - } - } - return (uint)(key ^ (key >> 32)); - } - public override ResourceOptions GetDefaultOptions () { return new QlieOptions { diff --git a/ArcFormats/Qlie/Encryption.cs b/ArcFormats/Qlie/Encryption.cs new file mode 100644 index 00000000..45e47c8f --- /dev/null +++ b/ArcFormats/Qlie/Encryption.cs @@ -0,0 +1,339 @@ +//! \file Encryption.cs +//! \date Sun Mar 12 04:51:11 2017 +//! \brief QLIE encryption-related classes. +// +// Copyright (C) 2017 by morkt +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// + +using System; +using System.Text; + +namespace GameRes.Formats.Qlie +{ + internal interface IEncryption + { + bool IsUnicode { get; } + + uint CalculateHash (byte[] data, int length); + + string DecryptName (byte[] name, int name_length); + + void DecryptEntry (byte[] data, int offset, int length, QlieEntry entry); + } + + internal abstract class QlieEncryption : IEncryption + { + public virtual bool IsUnicode { get { return false; } } + + /// + /// Hash generated from the key data contained within archive index. + /// + public uint ArcKey { get; protected set; } + + public static IEncryption Create (ArcView file, Version version, byte[] arc_key) + { + if (2 == version.Major || 1 == version.Major) + return new EncryptionV2(); + else if (3 == version.Major && 1 == version.Minor) + return new EncryptionV3_1 (file); + else if (3 == version.Major && 0 == version.Minor) + return new EncryptionV3 (file, arc_key); + else + throw new NotSupportedException ("Not supported QLIE archive version"); + } + + public abstract uint CalculateHash (byte[] data, int length); + + public abstract string DecryptName (byte[] name, int name_length); + + public abstract void DecryptEntry (byte[] data, int offset, int length, QlieEntry entry); + } + + internal class EncryptionV2 : QlieEncryption + { + public EncryptionV2 () + { + ArcKey = 0xC4; + } + + public override uint CalculateHash (byte[] data, int length) + { + return 0; // not implemented + } + + public override string DecryptName (byte[] name, int name_length) + { + int key = name_length + ((int)ArcKey ^ 0x3E); + for (int k = 0; k < name_length; ++k) + name[k] ^= (byte)(((k + 1) ^ key) + k + 1); + + return Encodings.cp932.GetString (name, 0, name_length); + } + + public override void DecryptEntry (byte[] data, int offset, int length, QlieEntry entry) + { + if (offset < 0) + throw new ArgumentOutOfRangeException ("offset"); + if (length > data.Length || offset > data.Length - length) + throw new ArgumentOutOfRangeException ("length"); + uint arc_key = 0; // ArcKey? + + ulong hash = 0xA73C5F9DA73C5F9Dul; + ulong xor = ((uint)length + arc_key) ^ 0xFEC9753Eu; + xor |= xor << 32; + unsafe + { + fixed (byte* raw = data) + { + ulong* encoded = (ulong*)(raw + offset); + for (int i = 0; i < length / 8; ++i) + { + hash = MMX.PAddD (hash, 0xCE24F523CE24F523ul) ^ xor; + xor = *encoded ^ hash; + *encoded++ = xor; + } + } + } + } + } + + internal class EncryptionV3 : EncryptionV2 + { + /// + /// Internal game data used to decrypt encrypted entries. + /// null if not used. + /// + public byte[] GameKeyData; + + public EncryptionV3 (ArcView file, byte[] game_key) + { + GameKeyData = game_key; + var key_data = file.View.ReadBytes (file.MaxOffset-0x41C, 0x100); + ArcKey = CalculateHash (key_data, key_data.Length) & 0x0FFFFFFFu; + } + + public override uint CalculateHash (byte[] data, int length) + { + if (length > data.Length) + throw new ArgumentOutOfRangeException ("length"); + unsafe + { + fixed (byte* data8 = data) + { + ulong hash = 0; + ulong key = 0; + ulong* data64 = (ulong*)data8; + for (int i = length / 8; i > 0; --i) + { + hash = MMX.PAddW (hash, 0x0307030703070307); + key = MMX.PAddW (key, *data64++ ^ hash); + } + return (uint)(key ^ (key >> 32)); + } + } + } + + public override void DecryptEntry (byte[] data, int offset, int length, QlieEntry entry) + { + var key_file = entry.KeyFile; + if (null == key_file) + { + base.DecryptEntry (data, offset, length, entry); + return; + } + // play it safe with 'unsafe' sections + if (offset < 0) + throw new ArgumentOutOfRangeException ("offset"); + if (length > data.Length || offset > data.Length - length) + throw new ArgumentOutOfRangeException ("length"); + + if (length < 8) + return; + + var file_name = entry.RawName; + uint hash = 0x85F532; + uint seed = 0x33F641; + + for (uint i = 0; i < file_name.Length; i++) + { + hash += (i & 0xFF) * file_name[i]; + seed ^= hash; + } + + seed += ArcKey ^ (7 * ((uint)length & 0xFFFFFF) + (uint)length + + hash + (hash ^ (uint)length ^ 0x8F32DCu)); + seed = 9 * (seed & 0xFFFFFF); + + if (GameKeyData != null) + seed ^= 0x453A; + + var mt = new QlieMersenneTwister (seed); + if (key_file != null) + mt.XorState (key_file); + if (GameKeyData != null) + mt.XorState (GameKeyData); + + // game code fills dword[41] table, but only the first 16 qwords are used + ulong[] table = new ulong[16]; + for (int i = 0; i < table.Length; ++i) + table[i] = mt.Rand64(); + + // compensate for 9 discarded dwords + for (int i = 0; i < 9; ++i) + mt.Rand(); + + ulong hash64 = mt.Rand64(); + uint t = mt.Rand() & 0xF; + unsafe + { + fixed (byte* raw_data = &data[offset]) + { + ulong* data64 = (ulong*)raw_data; + int qword_length = length / 8; + for (int i = 0; i < qword_length; ++i) + { + hash64 = MMX.PAddD (hash64 ^ table[t], table[t]); + + ulong d = data64[i] ^ hash64; + data64[i] = d; + + hash64 = MMX.PAddB (hash64, d) ^ d; + hash64 = MMX.PAddW (MMX.PSllD (hash64, 1), d); + + t++; + t &= 0xF; + } + } + } + } + } + + internal class EncryptionV3_1 : QlieEncryption + { + public override bool IsUnicode { get { return true; } } + + public EncryptionV3_1 (ArcView file) + { + var key_data = file.View.ReadBytes (file.MaxOffset-0x41C, 0x100); + ArcKey = CalculateHash (key_data, key_data.Length) & 0x0FFFFFFFu; + } + + public override uint CalculateHash (byte[] data, int length) + { + if (length > data.Length) + throw new ArgumentOutOfRangeException ("length"); + unsafe + { + fixed (byte* data8 = data) + { + ulong hash = 0; + ulong key = 0; + ulong* data64 = (ulong*)data8; + for (int i = length / 8; i > 0; --i) + { + hash = MMX.PAddW (hash, 0xA35793A7A35793A7ul); + key = MMX.PAddW (key, *data64++ ^ hash); + key = MMX.PSllD (key, 3) | MMX.PSrlD (key, 29); + } + // MMX.PMAddWD (key, key >> 32); + return (uint)((short)key * (short)(key >> 32) + (short)(key >> 16) * (short)(key >> 48)); + } + } + } + + public override string DecryptName (byte[] name, int name_length) + { + int char_count = name_length / 2; + int hash = (char_count * char_count) ^ char_count; + hash ^= (int)(0x3E13 ^ (ArcKey >> 16) ^ ArcKey); + hash &= 0xFFFF; + int key = hash; + for (int i = 0; i < char_count; ++i) + { + key = hash + i + 8 * key; + name[i*2 ] ^= (byte)key; + name[i*2+1] ^= (byte)(key >> 8); + } + return Encoding.Unicode.GetString (name, 0, name_length); + } + + public override void DecryptEntry (byte[] data, int offset, int length, QlieEntry entry) + { + if (offset < 0) + throw new ArgumentOutOfRangeException ("offset"); + if (length > data.Length || offset > data.Length - length) + throw new ArgumentOutOfRangeException ("length"); + if (length < 8) + return; + + var file_name = entry.Name; + uint hash = 0x85F532; + uint seed = 0x33F641; + + for (int i = 0; i < file_name.Length; i++) + { + hash += (uint)(file_name[i] << (i & 7)); + seed ^= hash; + } + + seed += ArcKey ^ (7 * ((uint)length & 0xFFFFFF) + (uint)length + + hash + (hash ^ (uint)length ^ 0x8F32DCu)); + seed = 9 * (seed & 0xFFFFFF); + var table = GenerateTable (0x20, seed); // originally 0x2C, 12 dwords not used + unsafe + { + fixed (byte* raw_data = &data[offset]) + { + ulong* data64 = (ulong*)raw_data; + int qword_length = length / 8; + uint k = 2 * (table[0xD] & 0xF); + ulong hash64 = table[6] | (ulong)table[7] << 32; + for (int i = 0; i < qword_length; ++i) + { + ulong t = table[k] | (ulong)table[k+1] << 32; + hash64 = MMX.PAddD (hash64 ^ t, t); + + ulong d = data64[i] ^ hash64; + data64[i] = d; + + hash64 = MMX.PAddB (hash64, d) ^ d; + hash64 = MMX.PAddW (MMX.PSllD (hash64, 1), d); + + k = (k + 2) & 0x1F; + } + } + } + } + + static uint[] GenerateTable (int length, uint seed) + { + const uint key = 0x8DF21431u; + var table = new uint[length]; + for (int i = 0; i < length; ++i) + { + ulong x = key * (ulong)(seed ^ key); + seed = (uint)((x >> 32) + x); + table[i] = seed; + } + return table; + } + } +}