//! \file ArcView.cs //! \date Mon Jul 07 10:31:10 2014 //! \brief Memory mapped view of gameres file. // // Copyright (C) 2014-2015 by morkt // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS // IN THE SOFTWARE. // using System; using System.IO; using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; using System.Text; using GameRes.Utility; namespace GameRes { public static class Encodings { public static readonly Encoding cp932 = Encoding.GetEncoding (932); public static Encoding WithFatalFallback (this Encoding enc) { var encoding = enc.Clone() as Encoding; encoding.EncoderFallback = EncoderFallback.ExceptionFallback; return encoding; } } public static class StreamExtension { static public string ReadStringUntil (this Stream file, byte delim, Encoding enc) { byte[] buffer = new byte[16]; int size = 0; for (;;) { int b = file.ReadByte (); if (-1 == b || delim == b) break; if (buffer.Length == size) { Array.Resize (ref buffer, checked(size/2*3)); } buffer[size++] = (byte)b; } return enc.GetString (buffer, 0, size); } static public string ReadCString (this Stream file, Encoding enc) { return ReadStringUntil (file, 0, enc); } static public string ReadCString (this Stream file) { return ReadStringUntil (file, 0, Encodings.cp932); } } public static class MappedViewExtension { static public string ReadString (this MemoryMappedViewAccessor view, long offset, uint size, Encoding enc) { if (0 == size) return string.Empty; byte[] buffer = new byte[size]; uint n; for (n = 0; n < size; ++n) { byte b = view.ReadByte (offset+n); if (0 == b) break; buffer[n] = b; } return enc.GetString (buffer, 0, (int)n); } static public string ReadString (this MemoryMappedViewAccessor view, long offset, uint size) { return ReadString (view, offset, size, Encodings.cp932); } unsafe public static byte* GetPointer (this MemoryMappedViewAccessor view, long offset) { var num = offset % info.dwAllocationGranularity; byte* ptr = null; view.SafeMemoryMappedViewHandle.AcquirePointer (ref ptr); ptr += num; return ptr; } [DllImport("kernel32.dll", SetLastError = true)] internal static extern void GetSystemInfo (ref SYSTEM_INFO lpSystemInfo); internal struct SYSTEM_INFO { internal int dwOemId; internal int dwPageSize; internal IntPtr lpMinimumApplicationAddress; internal IntPtr lpMaximumApplicationAddress; internal IntPtr dwActiveProcessorMask; internal int dwNumberOfProcessors; internal int dwProcessorType; internal int dwAllocationGranularity; internal short wProcessorLevel; internal short wProcessorRevision; } static SYSTEM_INFO info; static MappedViewExtension() { GetSystemInfo (ref info); } } public class ArcView : IDisposable { private MemoryMappedFile m_map; public const long PageSize = 4096; public long MaxOffset { get; private set; } public Frame View { get; private set; } public string Name { get; private set; } public ArcView (string name) { using (var fs = new FileStream(name, FileMode.Open, FileAccess.Read, FileShare.Read)) { Name = name; MaxOffset = fs.Length; InitFromFileStream (fs, 0); } } public ArcView (Stream input, string name, uint length) { Name = name; MaxOffset = length; if (input is FileStream) InitFromFileStream (input as FileStream, length); else InitFromStream (input, length); } private void InitFromFileStream (FileStream fs, uint length) { m_map = MemoryMappedFile.CreateFromFile (fs, null, length, MemoryMappedFileAccess.Read, null, HandleInheritability.None, true); try { View = new Frame (this); } catch { m_map.Dispose(); // dispose on error only throw; } } private void InitFromStream (Stream input, uint length) { m_map = MemoryMappedFile.CreateNew (null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, null, HandleInheritability.None); try { using (var view = m_map.CreateViewAccessor (0, length, MemoryMappedFileAccess.Write)) { var buffer = new byte[81920]; unsafe { byte* ptr = view.GetPointer (0); try { uint total = 0; while (total < length) { int read = input.Read (buffer, 0, buffer.Length); if (0 == read) break; read = (int)Math.Min (read, length-total); Marshal.Copy (buffer, 0, (IntPtr)(ptr+total), read); total += (uint)read; } MaxOffset = total; } finally { view.SafeMemoryMappedViewHandle.ReleasePointer(); } } } View = new Frame (this); } catch { m_map.Dispose(); throw; } } public Frame CreateFrame () { return new Frame (View); } public ArcStream CreateStream () { return new ArcStream (this); } public ArcStream CreateStream (long offset) { var size = this.MaxOffset - offset; if (size > uint.MaxValue) throw new ArgumentOutOfRangeException ("Too large memory mapped stream"); return new ArcStream (this, offset, (uint)size); } public ArcStream CreateStream (long offset, uint size) { return new ArcStream (this, offset, size); } public MemoryMappedViewAccessor CreateViewAccessor (long offset, uint size) { return m_map.CreateViewAccessor (offset, size, MemoryMappedFileAccess.Read); } #region IDisposable Members bool disposed = false; public void Dispose () { Dispose (true); GC.SuppressFinalize (this); } protected virtual void Dispose (bool disposing) { if (!disposed) { if (disposing) { View.Dispose(); m_map.Dispose(); } disposed = true; m_map = null; } } #endregion public class Frame : IDisposable { private ArcView m_arc; private MemoryMappedViewAccessor m_view; private long m_offset; private uint m_size; public long Offset { get { return m_offset; } } public uint Reserved { get { return m_size; } } public Frame (ArcView arc) { m_arc = arc; m_offset = 0; m_size = (uint)Math.Min (ArcView.PageSize, m_arc.MaxOffset); m_view = m_arc.CreateViewAccessor (m_offset, m_size); } public Frame (Frame other) { m_arc = other.m_arc; m_offset = 0; m_size = (uint)Math.Min (ArcView.PageSize, m_arc.MaxOffset); m_view = m_arc.CreateViewAccessor (m_offset, m_size); } public Frame (ArcView arc, long offset, uint size) { m_arc = arc; m_offset = Math.Min (offset, m_arc.MaxOffset); m_size = (uint)Math.Min (size, m_arc.MaxOffset-m_offset); m_view = m_arc.CreateViewAccessor (m_offset, m_size); } public uint Reserve (long offset, uint size) { if (offset < m_offset || offset+size > m_offset+m_size) { if (offset > m_arc.MaxOffset) throw new ArgumentOutOfRangeException ("offset", "Too large offset specified for memory mapped file view."); if (size < ArcView.PageSize) size = (uint)ArcView.PageSize; if (size > m_arc.MaxOffset-offset) size = (uint)(m_arc.MaxOffset-offset); var old_view = m_view; m_view = m_arc.CreateViewAccessor (offset, size); old_view.Dispose(); m_offset = offset; m_size = size; } return (uint)(m_offset + m_size - offset); } public bool AsciiEqual (long offset, string data) { if (Reserve (offset, (uint)data.Length) < (uint)data.Length) return false; unsafe { byte* ptr = m_view.GetPointer (m_offset); try { for (int i = 0; i < data.Length; ++i) { if (ptr[offset-m_offset+i] != data[i]) return false; } } finally { m_view.SafeMemoryMappedViewHandle.ReleasePointer(); } return true; } } public int Read (long offset, byte[] buf, int buf_offset, uint count) { // supposedly faster version of //Reserve (offset, count); //return m_view.ReadArray (offset-m_offset, buf, buf_offset, (int)count); if (buf == null) throw new ArgumentNullException ("buf", "Buffer cannot be null."); if (buf_offset < 0) throw new ArgumentOutOfRangeException ("buf_offset", "Buffer offset should be non-negative."); int total = (int)Math.Min (Reserve (offset, count), count); if (buf.Length - buf_offset < total) throw new ArgumentException ("Buffer offset and length are out of bounds."); unsafe { byte* ptr = m_view.GetPointer (m_offset); try { Marshal.Copy ((IntPtr)(ptr+(offset-m_offset)), buf, buf_offset, total); } finally { m_view.SafeMemoryMappedViewHandle.ReleasePointer(); } } return total; } /// /// Read bytes starting from into byte array and return that array. /// Returned array could be less than bytes length if end of the mapped file was reached. /// public byte[] ReadBytes (long offset, uint count) { count = Math.Min (count, Reserve (offset, count)); var data = new byte[count]; Read (offset, data, 0, count); return data; } public byte ReadByte (long offset) { Reserve (offset, 1); return m_view.ReadByte (offset-m_offset); } public sbyte ReadSByte (long offset) { Reserve (offset, 1); return m_view.ReadSByte (offset-m_offset); } public ushort ReadUInt16 (long offset) { Reserve (offset, 2); return m_view.ReadUInt16 (offset-m_offset); } public short ReadInt16 (long offset) { Reserve (offset, 2); return m_view.ReadInt16 (offset-m_offset); } public uint ReadUInt32 (long offset) { Reserve (offset, 4); return m_view.ReadUInt32 (offset-m_offset); } public int ReadInt32 (long offset) { Reserve (offset, 4); return m_view.ReadInt32 (offset-m_offset); } public ulong ReadUInt64 (long offset) { Reserve (offset, 8); return m_view.ReadUInt64 (offset-m_offset); } public long ReadInt64 (long offset) { Reserve (offset, 8); return m_view.ReadInt64 (offset-m_offset); } public string ReadString (long offset, uint size, Encoding enc) { size = Math.Min (size, Reserve (offset, size)); return m_view.ReadString (offset-m_offset, size, enc); /* unsafe implementation requires .Net v4.6 if (0 == size) return string.Empty; unsafe { byte* s = m_view.GetPointer (m_offset) + (offset - m_offset); try { uint string_length = 0; while (string_length < size && 0 != s[string_length]) { ++string_length; } return enc.GetString (s, (int)string_length); // .Net v4.6+ only } finally { m_view.SafeMemoryMappedViewHandle.ReleasePointer(); } } */ } public string ReadString (long offset, uint size) { return ReadString (offset, size, Encodings.cp932); } #region IDisposable Members bool disposed = false; public void Dispose () { Dispose (true); GC.SuppressFinalize (this); } protected virtual void Dispose (bool disposing) { if (!disposed) { if (disposing) { m_view.Dispose(); } m_arc = null; m_view = null; disposed = true; } } #endregion } public class ArcStream : Stream, IBinaryStream { private Frame m_view; private long m_start; private long m_size; private long m_position; public string Name { get; set; } public uint Signature { get { return ReadSignature(); } } public Stream AsStream { get { return this; } } public override bool CanRead { get { return !disposed; } } public override bool CanSeek { get { return !disposed; } } public override bool CanWrite { get { return false; } } public override long Length { get { return m_size; } } public override long Position { get { return m_position; } set { m_position = Math.Max (value, 0); } } public ArcStream (ArcView file) { m_view = file.CreateFrame(); m_start = 0; m_size = file.MaxOffset; m_position = 0; } public ArcStream (Frame view) { m_view = view; m_start = m_view.Offset; m_size = m_view.Reserved; m_position = 0; } public ArcStream (ArcView file, long offset, uint size) : this (new Frame (file, offset, size)) { } public ArcStream (Frame view, long offset, uint size) { m_view = view; m_start = offset; m_size = Math.Min (size, m_view.Reserve (offset, size)); m_position = 0; } /// /// Read stream signature (first 4 bytes) without altering current read position. /// public uint ReadSignature () { return m_view.ReadUInt32 (m_start); } byte[] m_header; int m_header_size; public CowArray ReadHeader (int size) { if (m_header_size < size) { if (null == m_header || m_header.Length < size) Array.Resize (ref m_header, (size + 0xF) & ~0xF); long position = m_start + m_header_size; m_header_size += m_view.Read (position, m_header, m_header_size, (uint)(size - m_header_size)); } size = Math.Min (size, m_header_size); Position = size; return new CowArray (m_header, 0, size); } public int PeekByte () { if (m_position >= m_size) return -1; return m_view.ReadByte (m_start+m_position); } public override int ReadByte () { int b = PeekByte(); if (-1 != b) ++m_position; return b; } public sbyte ReadSByte () { int b = ReadByte(); if (-1 == b) throw new EndOfStreamException(); return (sbyte)b; } public short ReadInt16 () { if (m_position + 2 > m_size) throw new EndOfStreamException(); var v = m_view.ReadInt16 (m_start+m_position); m_position += 2; return v; } public ushort ReadUInt16 () { return (ushort)ReadInt16(); } public int ReadInt24 () { if (m_position + 3 > m_size) throw new EndOfStreamException(); int v = m_view.ReadUInt16 (m_start+m_position); v |= m_view.ReadByte (m_start+m_position+2); m_position += 3; return v; } public int ReadInt32 () { if (m_position + 4 > m_size) throw new EndOfStreamException(); var v = m_view.ReadInt32 (m_start+m_position); m_position += 4; return v; } public uint ReadUInt32 () { return (uint)ReadInt32(); } public long ReadInt64 () { if (m_position + 8 > m_size) throw new EndOfStreamException(); var v = m_view.ReadInt64 (m_start+m_position); m_position += 8; return v; } public ulong ReadUInt64 () { return (ulong)ReadInt64(); } byte[] m_string_buf; public string ReadCString (int length) { return ReadCString (length, Encodings.cp932); } public string ReadCString (int length, Encoding enc) { if (null == m_string_buf || m_string_buf.Length < length) m_string_buf = new byte[Math.Max (length, 0x20)]; length = Read (m_string_buf, 0, length); return Binary.GetCString (m_string_buf, 0, length); } public string ReadCString () { return ReadCString (Encodings.cp932); } public string ReadCString (Encoding enc) { if (null == m_string_buf) m_string_buf = new byte[0x20]; int size = 0; for (;;) { int b = ReadByte(); if (-1 == b || 0 == b) break; if (m_string_buf.Length == size) { Array.Resize (ref m_string_buf, checked(size*3/2)); } m_string_buf[size++] = (byte)b; } return enc.GetString (m_string_buf, 0, size); } public byte[] ReadBytes (int count) { if (m_position >= m_size) return new byte[0]; var bytes = m_view.ReadBytes (m_start+m_position, (uint)Math.Min (count, m_size - m_position)); m_position += bytes.Length; return bytes; } #region System.IO.Stream methods public override void Flush() { } public override long Seek (long offset, SeekOrigin origin) { switch (origin) { case SeekOrigin.Begin: m_position = offset; break; case SeekOrigin.Current: m_position += offset; break; case SeekOrigin.End: m_position = m_size + offset; break; } if (m_position < 0) m_position = 0; return m_position; } public override void SetLength (long length) { throw new NotSupportedException ("GameRes.ArcStream.SetLength method is not supported"); } public override int Read (byte[] buffer, int offset, int count) { if (m_position >= m_size) return 0; count = (int)Math.Min (count, m_size - m_position); int read = m_view.Read (m_start + m_position, buffer, offset, (uint)count); m_position += read; return read; } public override void Write (byte[] buffer, int offset, int count) { throw new NotSupportedException("GameRes.ArcStream.Write method is not supported"); } public override void WriteByte (byte value) { throw new NotSupportedException("GameRes.ArcStream.WriteByte method is not supported"); } #endregion #region IDisposable Members bool disposed = false; protected override void Dispose (bool disposing) { if (!disposed) { if (disposing) { m_view.Dispose(); } disposed = true; base.Dispose (disposing); } } #endregion } public class Reader : System.IO.BinaryReader { public Reader (Stream stream) : base (stream, Encoding.ASCII, true) { } } } /// /// Unsafe wrapper around unmanaged memory mapped view pointer. /// public unsafe class ViewPointer : IDisposable { MemoryMappedViewAccessor m_view; byte* m_ptr; public ViewPointer (MemoryMappedViewAccessor view, long offset) { m_view = view; m_ptr = m_view.GetPointer (offset); } public byte* Value { get { return m_ptr; } } #region IDisposable Members bool _disposed = false; public void Dispose () { Dispose (true); GC.SuppressFinalize (this); } protected virtual void Dispose (bool disposing) { if (!_disposed) { if (disposing) { m_view.SafeMemoryMappedViewHandle.ReleasePointer(); } _disposed = true; } } #endregion } }