implemented 'Nemesis' compression.

ArcFormats/ArcFormats.csproj

@@ -84,6 +84,7 @@
     <Compile Include="Cmvs\ImagePB3.cs" />
     <Compile Include="DenSDK\ArcDAF.cs" />
     <Compile Include="elf\ArcVSD.cs" />
+    <Compile Include="Entis\ErisaNemesis.cs" />
     <Compile Include="Escude\ArcBIN.cs" />
     <Compile Include="Eushully\ArcGPC.cs" />
     <Compile Include="Eushully\ImageGP.cs" />
@@ -544,7 +545,9 @@
     </EmbeddedResource>
     <EmbeddedResource Include="Strings\arcStrings.ru-RU.resx" />
   </ItemGroup>
-  <ItemGroup />
+  <ItemGroup>
+    <Folder Include="CsWare\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <PreBuildEvent>perl "$(SolutionDir)inc-revision.pl" "$(ProjectPath)" $(ConfigurationName)

ArcFormats/Entis/ArcNOA.cs

@@ -123,56 +123,61 @@ namespace GameRes.Formats.Entis
             ulong size = arc.File.View.ReadUInt64 (entry.Offset+8);
             if (size > int.MaxValue)
                 throw new FileSizeException();
-            if (0 == size)
+            if (size <= 4)
                 return Stream.Null;
 
+            if (EncType.ERISACode == nent.Encryption)
+            {
+                using (var enc = arc.File.CreateStream (entry.Offset+0x10, (uint)size-4))
+                    return DecodeNemesis (enc);
+            }
+
+            var narc = arc as NoaArchive;
             var input = arc.File.CreateStream (entry.Offset+0x10, (uint)size);
-            try
+            if (EncType.Raw == nent.Encryption || null == narc || null == narc.Password)
-            {
+                return input;
-                var narc = arc as NoaArchive;
-                if (0 == nent.Encryption || size < 4 || null == narc || null == narc.Password)
-                    return input;
-                if (0x40000000 != nent.Encryption)
-                {
-                    Trace.WriteLine (string.Format ("{0}: unknown encryption scheme 0x{1:x8}",
-                                                    nent.Name, nent.Encryption));
-                    return input;
-                }
-                uint nTotalBytes = (uint)(size - 4);
-                var pBSHF = new BSHFDecodeContext (0x10000);
-                pBSHF.AttachInputFile (input);
-                pBSHF.PrepareToDecodeBSHFCode (narc.Password);
 
-                byte[] buf = new byte[nTotalBytes];
+            if (EncType.BSHFCrypt == nent.Encryption)
-                uint decoded = pBSHF.DecodeBSHFCodeBytes (buf, nTotalBytes);
-                if (decoded < nTotalBytes)
-                    throw new EndOfStreamException ("Unexpected end of encrypted stream");
-
-                /* Something wrong with preceding length calculation, resulting CRC doesn't match
-                byte[] bufCRC = new byte[4];
-                int iCRC = 0;
-                for (int i = 0; i < buf.Length; ++i)
-                {
-                    bufCRC[iCRC] ^= buf[i];
-                    iCRC = (iCRC + 1) & 0x03;
-                }
-                uint orgCRC = arc.File.View.ReadUInt32 (entry.Offset+0x10+nTotalBytes);
-                uint crc = LittleEndian.ToUInt32 (bufCRC, 0);
-                if (orgCRC != crc)
-                {
-                    Trace.WriteLine (string.Format ("{0}: CRC mismatch", nent.Name));
-                    input.Position = 0;
-                    return input;
-                }
-                */
-                input.Dispose();
-                return new MemoryStream (buf);
-            }
-            catch
             {
-                input.Dispose();
+                using (input)
-                throw;
+                    return DecodeBSHF (input, narc.Password);
             }
+            Trace.WriteLine (string.Format ("{0}: encryption scheme 0x{1:x8} not implemented",
+                                            nent.Name, nent.Encryption));
+            return input;
+        }
+
+        Stream DecodeNemesis (Stream input)
+        {
+            var decoder = new NemesisDecodeContext();
+            decoder.AttachInputFile (input);
+            decoder.PrepareToDecodeERISANCode();
+            var file = new MemoryStream ((int)input.Length);
+            var buffer = new byte[0x10000];
+            for (;;)
+            {
+                int read = (int)decoder.DecodeNemesisCodeBytes (buffer, 0x10000);
+                if (0 == read)
+                    break;
+                file.Write (buffer, 0, read);
+            }
+            file.Position = 0;
+            return file;
+        }
+
+        Stream DecodeBSHF (Stream input, string password)
+        {
+            uint nTotalBytes = (uint)input.Length - 4;
+            var pBSHF = new BSHFDecodeContext (0x10000);
+            pBSHF.AttachInputFile (input);
+            pBSHF.PrepareToDecodeBSHFCode (password);
+
+            byte[] buf = new byte[nTotalBytes];
+            uint decoded = pBSHF.DecodeBSHFCodeBytes (buf, nTotalBytes);
+            if (decoded < nTotalBytes)
+                throw new EndOfStreamException ("Unexpected end of encrypted stream");
+
+            return new MemoryStream (buf);
         }
 
         public override ResourceOptions GetDefaultOptions ()
@@ -188,6 +193,16 @@ namespace GameRes.Formats.Entis
             return new GUI.WidgetNOA();
         }
 
+        internal static class EncType
+        {
+            public const uint Raw           = 0x00000000;
+            public const uint ERISACode     = 0x80000010;
+            public const uint BSHFCrypt     = 0x40000000;
+            public const uint SimpleCrypt32 = 0x20000000;
+            public const uint ERISACrypt    = 0xC0000010;
+            public const uint ERISACrypt32  = 0xA0000010;
+        }
+
         internal class IndexReader
         {
             ArcView     m_file;
@@ -235,8 +250,7 @@ namespace GameRes.Formats.Entis
                     dir_offset += 4;
 
                     entry.Encryption = m_file.View.ReadUInt32 (dir_offset);
-                    if (0 != entry.Encryption)
+                    m_found_encrypted = m_found_encrypted || (EncType.Raw != entry.Encryption && EncType.ERISACode != entry.Encryption);
-                        m_found_encrypted = true;
                     dir_offset += 4;
 
                     entry.Offset = base_offset + m_file.View.ReadInt64 (dir_offset);

ArcFormats/Entis/EriReader.cs

@@ -1368,16 +1368,16 @@ namespace GameRes.Formats.Entis
 
     internal class ProbDecodeContext : RLEDecodeContext
     {
-        uint        m_dwCodeRegister;
+        protected uint      m_dwCodeRegister;
-        uint        m_dwAugendRegister;
+        protected uint      m_dwAugendRegister;
-        int         m_nPostBitCount;
+        protected int       m_nPostBitCount;
-        byte[]      m_bytLastSymbol = new byte[4];
+        protected byte[]    m_bytLastSymbol = new byte[4];
 
-        ErisaProbModel          m_pPhraseLenProb = new ErisaProbModel();
+        protected ErisaProbModel m_pPhraseLenProb = new ErisaProbModel();
-        ErisaProbModel          m_pPhraseIndexProb = new ErisaProbModel();
+        protected ErisaProbModel m_pPhraseIndexProb = new ErisaProbModel();
-        ErisaProbModel          m_pRunLenProb = new ErisaProbModel();
+        protected ErisaProbModel m_pRunLenProb = new ErisaProbModel();
-        ErisaProbModel          m_pLastERISAProb;
+        protected ErisaProbModel m_pLastERISAProb;
-        ErisaProbModel[]        m_ppTableERISA;
+        protected ErisaProbModel[] m_ppTableERISA;
 
         public ProbDecodeContext (uint nBufferingSize) : base (nBufferingSize)
         {
@@ -1467,7 +1467,7 @@ namespace GameRes.Formats.Entis
             return nSymbol;
         }
 
-        int DecodeERISACodeIndex (ErisaProbModel pModel)
+        protected int DecodeERISACodeIndex (ErisaProbModel pModel)
         {
             uint dwAcc = m_dwCodeRegister * pModel.TotalCount / m_dwAugendRegister;
             if (dwAcc >= ErisaProbModel.TotalLimit)
@@ -1499,7 +1499,7 @@ namespace GameRes.Formats.Entis
                 {
                     if ((++m_nPostBitCount) >= 256)
                         return -1;
-                    nNextBit = 0 ;
+                    nNextBit = 0;
                 }
                 m_dwCodeRegister = (m_dwCodeRegister << 1) | ((uint)nNextBit & 1);
                 m_dwAugendRegister <<= 1;

ArcFormats/Entis/ErisaNemesis.cs

@@ -0,0 +1,293 @@
+//! \file       ErisaNemesis.cs
+//! \date       Tue Jan 19 18:24:23 2016
+//! \brief      Erisa Nemesis encoding implementation.
+//
+
+using System;
+using System.Diagnostics;
+
+namespace GameRes.Formats.Entis
+{
+    class NemesisDecodeContext : ProbDecodeContext
+    {
+        int         m_iLastSymbol;
+        int         m_nNemesisLeft;
+        int         m_nNemesisNext;
+        byte[]      m_pNemesisBuf;
+        int         m_nNemesisIndex;
+        bool        m_flagEOF;
+
+        ErisaProbBase           m_pProbERISA;
+        NemesisPhraseLookup[]   m_pNemesisLookup;
+
+        public NemesisDecodeContext (uint buffer_size = 0x10000) : base (buffer_size)
+        {
+            m_flagEOF = false;
+        }
+
+        public void PrepareToDecodeERISANCode ()
+        {
+            if (null == m_pProbERISA)
+                m_pProbERISA = new ErisaProbBase();
+
+            m_iLastSymbol = 0;
+            for (int i = 0; i < 4; ++i)
+                m_bytLastSymbol[i] = 0;
+
+            m_pProbERISA.dwWorkUsed = 0;
+            m_pProbERISA.epmBaseModel.Initialize();
+
+            for (int i = 0; i < ErisaProbBase.SlotMax; ++i)
+            {
+                m_pProbERISA.ptrProbIndex[i] = new ErisaProbModel();
+            }
+            PrepareToDecodeERISACode();
+
+            if (null == m_pNemesisBuf)
+            {
+                m_pNemesisBuf = new byte[Nemesis.BufSize];
+            }
+            if (null == m_pNemesisLookup)
+            {
+                m_pNemesisLookup = new NemesisPhraseLookup[0x100];
+            }
+            for (int i = 0; i < m_pNemesisBuf.Length; ++i)
+                m_pNemesisBuf[i] = 0;
+            for (int i = 0; i < m_pNemesisLookup.Length; ++i)
+                m_pNemesisLookup[i] = new NemesisPhraseLookup();
+            m_nNemesisIndex = 0;
+
+            m_nNemesisLeft = 0;
+            m_flagEOF = false;
+        }
+
+        public override uint DecodeBytes (Array ptrDst, uint nCount)
+        {
+            return DecodeNemesisCodeBytes (ptrDst as byte[], nCount);
+        }
+
+        public uint DecodeNemesisCodeBytes (byte[] ptrDst, uint nCount)
+        {
+            if (m_flagEOF)
+                return 0;
+
+            ErisaProbBase pBase = m_pProbERISA;
+            uint nDecoded = 0;
+            int dst = 0;
+            byte bytSymbol;
+            while (nDecoded < nCount)
+            {
+                if (m_nNemesisLeft > 0)
+                {
+                    uint nNemesisCount = (uint)m_nNemesisLeft;
+                    if (nNemesisCount > nCount - nDecoded)
+                    {
+                        nNemesisCount = nCount - nDecoded;
+                    }
+                    byte bytLastSymbol = m_pNemesisBuf[(m_nNemesisIndex - 1) & Nemesis.BufMask];
+
+                    for (uint i = 0; i < nNemesisCount; ++i)
+                    {
+                        bytSymbol = bytLastSymbol;
+                        if (m_nNemesisNext >= 0)
+                        {
+                            bytSymbol = m_pNemesisBuf[m_nNemesisNext++];
+                            m_nNemesisNext &= Nemesis.BufMask;
+                        }
+                        m_bytLastSymbol[m_iLastSymbol++] = bytSymbol;
+                        m_iLastSymbol &= 3;
+
+                        var phrase = m_pNemesisLookup[bytSymbol];
+                        phrase.index[phrase.first] = (uint)m_nNemesisIndex;
+                        phrase.first = (phrase.first + 1) & Nemesis.IndexMask;
+                        bytLastSymbol = bytSymbol;
+
+                        m_pNemesisBuf[m_nNemesisIndex++] = bytSymbol;
+                        m_nNemesisIndex &= Nemesis.BufMask;
+
+                        ptrDst[dst++] = bytSymbol;
+                    }
+                    m_nNemesisLeft -= (int)nNemesisCount;
+                    nDecoded += nNemesisCount;
+                    continue;
+                }
+
+                int iDeg;
+                ErisaProbModel pModel = pBase.epmBaseModel;
+                for (iDeg = 0; iDeg < 4; ++iDeg)
+                {
+                    int iLast = m_bytLastSymbol[(m_iLastSymbol + 3 - iDeg) & 3]
+                                >> ErisaProbBase.m_nShiftCount[iDeg];
+                    if (pModel.SubModel[iLast].Symbol < 0)
+                    {
+                        break;
+                    }
+                    Debug.Assert ((uint)pModel.SubModel[iLast].Symbol < pBase.dwWorkUsed);
+                    pModel = pBase.ptrProbIndex[pModel.SubModel[iLast].Symbol];
+                }
+                int iSym = DecodeERISACodeIndex (pModel);
+                if (iSym < 0)
+                {
+                    return nDecoded;
+                }
+                int nSymbol = pModel.SymTable[iSym].Symbol;
+                int iSymIndex = pModel.IncreaseSymbol (iSym);
+
+                bool fNemesis = false;
+                if (nSymbol == ErisaProbModel.EscCode)
+                {
+                    if (pModel != pBase.epmBaseModel)
+                    {
+                        iSym = DecodeERISACodeIndex (pBase.epmBaseModel);
+                        if (iSym < 0)
+                        {
+                            return  nDecoded;
+                        }
+                        nSymbol = pBase.epmBaseModel.SymTable[iSym].Symbol;
+                        pBase.epmBaseModel.IncreaseSymbol (iSym);
+                        if (nSymbol != ErisaProbModel.EscCode)
+                        {
+                            pModel.AddSymbol ((short)nSymbol);
+                            iSym = -1;
+                        }
+                        else
+                        {
+                            fNemesis = true;
+                        }
+                    }
+                    else
+                    {
+                        fNemesis = true;
+                    }
+                }
+                if (fNemesis)
+                {
+                    int nLength, nPhraseIndex;
+                    nPhraseIndex = DecodeERISACode (m_pPhraseIndexProb);
+                    if (nPhraseIndex == ErisaProbModel.EscCode)
+                    {
+                        m_flagEOF = true;
+                        return nDecoded;
+                    }
+                    if (0 == nPhraseIndex)
+                    {
+                        nLength = DecodeERISACode (m_pRunLenProb);
+                    }
+                    else
+                    {
+                        nLength = DecodeERISACode (m_pPhraseLenProb);
+                    }
+                    if (nLength == ErisaProbModel.EscCode)
+                    {
+                        return  nDecoded;
+                    }
+                    byte bytLastSymbol = m_pNemesisBuf[(m_nNemesisIndex - 1) & Nemesis.BufMask];
+                    var phrase = m_pNemesisLookup[bytLastSymbol];
+                    m_nNemesisLeft = nLength;
+                    if (0 == nPhraseIndex)
+                    {
+                        m_nNemesisNext = -1;
+                    }
+                    else
+                    {
+                        m_nNemesisNext = (int)phrase.index[(phrase.first - nPhraseIndex) & Nemesis.IndexMask];
+                        Debug.Assert (m_pNemesisBuf[m_nNemesisNext] == bytLastSymbol);
+                        m_nNemesisNext = (m_nNemesisNext + 1) & Nemesis.BufMask;
+                    }
+                    continue;
+                }
+                bytSymbol = (byte)nSymbol;
+                m_bytLastSymbol[m_iLastSymbol++] = bytSymbol;
+                m_iLastSymbol &= 3;
+
+                var ppl = m_pNemesisLookup[bytSymbol];
+                ppl.index[ppl.first] = (uint)m_nNemesisIndex;
+                ppl.first = (ppl.first + 1) & Nemesis.IndexMask;
+                m_pNemesisBuf[m_nNemesisIndex++] = bytSymbol;
+                m_nNemesisIndex &= Nemesis.BufMask;
+
+                ptrDst[dst++] = bytSymbol;
+                nDecoded++;
+
+                if ((pBase.dwWorkUsed < ErisaProbBase.SlotMax) && (iDeg < 4))
+                {
+                    int iSymbol = ((byte)nSymbol) >> ErisaProbBase.m_nShiftCount[iDeg];
+                    Debug.Assert (iSymbol < ErisaProbModel.SubSortMax);
+                    if (++pModel.SubModel[iSymbol].Occured >= ErisaProbBase.m_nNewProbLimit[iDeg])
+                    {
+                        int i;
+                        ErisaProbModel pParent = pModel;
+                        pModel = pBase.epmBaseModel;
+                        for (i = 0; i <= iDeg; ++i)
+                        {
+                            iSymbol = m_bytLastSymbol[(m_iLastSymbol + 3 - i) & 3]
+                                      >> ErisaProbBase.m_nShiftCount[i];
+                            if (pModel.SubModel[iSymbol].Symbol < 0)
+                            {
+                                break;
+                            }
+                            Debug.Assert ((uint)pModel.SubModel[iSymbol].Symbol < pBase.dwWorkUsed);
+                            pModel = pBase.ptrProbIndex[pModel.SubModel[iSymbol].Symbol];
+                        }
+                        if ((i <= iDeg) && (pModel.SubModel[iSymbol].Symbol < 0))
+                        {
+                            ErisaProbModel pNew = pBase.ptrProbIndex[pBase.dwWorkUsed];
+                            pModel.SubModel[iSymbol].Symbol = (short)(pBase.dwWorkUsed++);
+
+                            pNew.TotalCount = 0;
+                            int j = 0;
+                            for (i = 0; i < (int)pParent.SymbolSorts; ++i)
+                            {
+                                ushort wOccured = (ushort)(pParent.SymTable[i].Occured >> 4);
+                                if (wOccured > 0 && (pParent.SymTable[i].Symbol != ErisaProbModel.EscCode))
+                                {
+                                    pNew.TotalCount += wOccured;
+                                    pNew.SymTable[j].Occured = wOccured;
+                                    pNew.SymTable[j].Symbol = pParent.SymTable[i].Symbol;
+                                    j++;
+                                }
+                            }
+                            pNew.TotalCount++;
+                            pNew.SymTable[j].Occured = 1;
+                            pNew.SymTable[j].Symbol = ErisaProbModel.EscCode;
+                            pNew.SymbolSorts = ++j;
+
+                            for (i = 0; i < ErisaProbModel.SubSortMax; ++i)
+                            {
+                                pNew.SubModel[i].Occured = 0;
+                                pNew.SubModel[i].Symbol = -1;
+                            }
+                        }
+                    }
+                }
+            }
+            return nDecoded;
+        }
+    }
+
+    internal class ErisaProbBase
+    {
+        public uint                 dwWorkUsed;
+        public ErisaProbModel       epmBaseModel = new ErisaProbModel();
+        public ErisaProbModel[]     ptrProbIndex = new ErisaProbModel[SlotMax];
+
+        public const int SlotMax = 0x800;
+
+        public static readonly int[] m_nShiftCount = { 1, 3, 4, 5 };
+        public static readonly int[] m_nNewProbLimit = { 0x01, 0x08, 0x10, 0x20 };
+    }
+
+    internal static class Nemesis
+    {
+        public const int BufSize = 0x10000;
+        public const int BufMask = 0xFFFF;
+        public const int IndexLimit  = 0x100;
+        public const int IndexMask   = 0xFF;
+    }
+
+    internal class NemesisPhraseLookup
+    {
+        public uint     first;
+        public uint[]   index = new uint[Nemesis.IndexLimit];
+    }
+}