diff --git a/Experimental/Artemis/GplexBuffers.cs b/Experimental/Artemis/GplexBuffers.cs new file mode 100644 index 00000000..6c2152e5 --- /dev/null +++ b/Experimental/Artemis/GplexBuffers.cs @@ -0,0 +1,720 @@ +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + public class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + public abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + public delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + public static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/Experimental/Artemis/IPT.Language.analyzer.lex b/Experimental/Artemis/IPT.Language.analyzer.lex new file mode 100644 index 00000000..6b1e5e95 --- /dev/null +++ b/Experimental/Artemis/IPT.Language.analyzer.lex @@ -0,0 +1,33 @@ +%namespace GameRes.Formats.Artemis +%scannertype IPTScanner +%visibility internal +%tokentype Token + +%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers + +Space [ \t\v\n\f] +Number [0-9]+ + +%{ + +%} + +%% + +{Space}+ /* skip */ + +{Number} { GetNumber(); return (int)Token.NUMBER; } + +\"(\\.|[^\\"\n])*\" { GetStringLiteral(); return (int)Token.STRING_LITERAL; } + +[a-zA-Z]+ { yylval.s = yytext; return (int)Token.IDENTIFIER; } + +"{" { return '{'; } + +"}" { return '}'; } + +"=" { return '='; } + +"," { return ','; } + +%% \ No newline at end of file diff --git a/Experimental/Artemis/IPT.Language.grammar.y b/Experimental/Artemis/IPT.Language.grammar.y new file mode 100644 index 00000000..51fd97ae --- /dev/null +++ b/Experimental/Artemis/IPT.Language.grammar.y @@ -0,0 +1,49 @@ +%namespace GameRes.Formats.Artemis +%partial +%parsertype IPTParser +%visibility internal +%tokentype Token + +%union { + public int n; + public string s; + public IPTObject o; +} + +%start input + +%token NUMBER STRING_LITERAL IDENTIFIER + +%% + +input: root_definition ; + +root_definition: IDENTIFIER '=' object { RootObject[$1.s] = $3.o; } + ; + +object: '{' { BeginObject(); } + decl_list optional_comma + '}' { EndObject(); } + ; + +decl_list: statement + | decl_list ',' statement + ; + +optional_comma: ',' | /* empty */ ; + +statement: definition | lvalue ; + +definition: IDENTIFIER '=' value { CurrentObject[$1.s] = $3.Value; } + ; + +lvalue: value { CurrentObject.Values.Add ($1.Value); } + ; + +value: object | string | number ; + +string: STRING_LITERAL ; + +number: NUMBER ; + +%% \ No newline at end of file diff --git a/Experimental/Artemis/IPT.Parser.Generated.cs b/Experimental/Artemis/IPT.Parser.Generated.cs new file mode 100644 index 00000000..ebe6f3ab --- /dev/null +++ b/Experimental/Artemis/IPT.Parser.Generated.cs @@ -0,0 +1,151 @@ +// This code was generated by the Gardens Point Parser Generator +// Copyright (c) Wayne Kelly, John Gough, QUT 2005-2014 +// (see accompanying GPPGcopyright.rtf) + +// GPPG version 1.5.2 + +// options: no-lines gplex + +using System; +using System.Collections.Generic; +using System.CodeDom.Compiler; +using System.Globalization; +using System.Text; +using QUT.Gppg; + +namespace GameRes.Formats.Artemis +{ +internal enum Token { + error=127,EOF=128,NUMBER=129,STRING_LITERAL=130,IDENTIFIER=131}; + +internal partial struct ValueType +{ + public int n; + public string s; + public IPTObject o; +} +// Abstract base class for GPLEX scanners +[GeneratedCodeAttribute( "Gardens Point Parser Generator", "1.5.2")] +internal abstract class ScanBase : AbstractScanner { + private LexLocation __yylloc = new LexLocation(); + public override LexLocation yylloc { get { return __yylloc; } set { __yylloc = value; } } + protected virtual bool yywrap() { return true; } +} + +// Utility class for encapsulating token information +[GeneratedCodeAttribute( "Gardens Point Parser Generator", "1.5.2")] +internal class ScanObj { + public int token; + public ValueType yylval; + public LexLocation yylloc; + public ScanObj( int t, ValueType val, LexLocation loc ) { + this.token = t; this.yylval = val; this.yylloc = loc; + } +} + +[GeneratedCodeAttribute( "Gardens Point Parser Generator", "1.5.2")] +internal partial class IPTParser: ShiftReduceParser +{ +#pragma warning disable 649 + private static Dictionary aliases; +#pragma warning restore 649 + private static Rule[] rules = new Rule[19]; + private static State[] states = new State[26]; + private static string[] nonTerms = new string[] { + "input", "$accept", "root_definition", "object", "Anon@1", "decl_list", + "optional_comma", "statement", "definition", "lvalue", "value", "string", + "number", }; + + static IPTParser() { + states[0] = new State(new int[]{131,4},new int[]{-1,1,-3,3}); + states[1] = new State(new int[]{128,2}); + states[2] = new State(-1); + states[3] = new State(-2); + states[4] = new State(new int[]{61,5}); + states[5] = new State(new int[]{123,7},new int[]{-4,6}); + states[6] = new State(-3); + states[7] = new State(-4,new int[]{-5,8}); + states[8] = new State(new int[]{131,15,123,7,130,20,129,22},new int[]{-6,9,-8,25,-9,14,-10,23,-11,24,-4,18,-12,19,-13,21}); + states[9] = new State(new int[]{44,12,125,-9},new int[]{-7,10}); + states[10] = new State(new int[]{125,11}); + states[11] = new State(-5); + states[12] = new State(new int[]{131,15,123,7,130,20,129,22,125,-8},new int[]{-8,13,-9,14,-10,23,-11,24,-4,18,-12,19,-13,21}); + states[13] = new State(-7); + states[14] = new State(-10); + states[15] = new State(new int[]{61,16}); + states[16] = new State(new int[]{123,7,130,20,129,22},new int[]{-11,17,-4,18,-12,19,-13,21}); + states[17] = new State(-12); + states[18] = new State(-14); + states[19] = new State(-15); + states[20] = new State(-17); + states[21] = new State(-16); + states[22] = new State(-18); + states[23] = new State(-11); + states[24] = new State(-13); + states[25] = new State(-6); + + for (int sNo = 0; sNo < states.Length; sNo++) states[sNo].number = sNo; + + rules[1] = new Rule(-2, new int[]{-1,128}); + rules[2] = new Rule(-1, new int[]{-3}); + rules[3] = new Rule(-3, new int[]{131,61,-4}); + rules[4] = new Rule(-5, new int[]{}); + rules[5] = new Rule(-4, new int[]{123,-5,-6,-7,125}); + rules[6] = new Rule(-6, new int[]{-8}); + rules[7] = new Rule(-6, new int[]{-6,44,-8}); + rules[8] = new Rule(-7, new int[]{44}); + rules[9] = new Rule(-7, new int[]{}); + rules[10] = new Rule(-8, new int[]{-9}); + rules[11] = new Rule(-8, new int[]{-10}); + rules[12] = new Rule(-9, new int[]{131,61,-11}); + rules[13] = new Rule(-10, new int[]{-11}); + rules[14] = new Rule(-11, new int[]{-4}); + rules[15] = new Rule(-11, new int[]{-12}); + rules[16] = new Rule(-11, new int[]{-13}); + rules[17] = new Rule(-12, new int[]{130}); + rules[18] = new Rule(-13, new int[]{129}); + } + + protected override void Initialize() { + this.InitSpecialTokens((int)Token.error, (int)Token.EOF); + this.InitStates(states); + this.InitRules(rules); + this.InitNonTerminals(nonTerms); + } + + protected override void DoAction(int action) + { +#pragma warning disable 162, 1522 + switch (action) + { + case 3: // root_definition -> IDENTIFIER, '=', object +{ RootObject[ValueStack[ValueStack.Depth-3].s] = ValueStack[ValueStack.Depth-1].o; } + break; + case 4: // Anon@1 -> /* empty */ +{ BeginObject(); } + break; + case 5: // object -> '{', Anon@1, decl_list, optional_comma, '}' +{ EndObject(); } + break; + case 12: // definition -> IDENTIFIER, '=', value +{ CurrentObject[ValueStack[ValueStack.Depth-3].s] = ValueStack[ValueStack.Depth-1].Value; } + break; + case 13: // lvalue -> value +{ CurrentObject.Values.Add (ValueStack[ValueStack.Depth-1].Value); } + break; + } +#pragma warning restore 162, 1522 + } + + protected override string TerminalToString(int terminal) + { + if (aliases != null && aliases.ContainsKey(terminal)) + return aliases[terminal]; + else if (((Token)terminal).ToString() != terminal.ToString(CultureInfo.InvariantCulture)) + return ((Token)terminal).ToString(); + else + return CharToString((char)terminal); + } + +} +} diff --git a/Experimental/Artemis/IPT.Parser.cs b/Experimental/Artemis/IPT.Parser.cs new file mode 100644 index 00000000..b4f42f6a --- /dev/null +++ b/Experimental/Artemis/IPT.Parser.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace GameRes.Formats.Artemis +{ + internal partial struct ValueType + { + public object Value { get { return o ?? s as object ?? (object)n; } } + } + + internal partial class IPTParser + { + public IPTParser () : base (null) { } + + internal IPTObject RootObject { get; set; } + + Stack m_value_stack; + + public void Parse (Stream s) + { + this.RootObject = new IPTObject(); + m_value_stack = new Stack(); + this.Scanner = new IPTScanner (s); + this.Parse(); + } + + internal IPTObject CurrentObject + { + get { return m_value_stack.Count > 0 ? m_value_stack.Peek() : RootObject; } + } + + void BeginObject () + { + m_value_stack.Push (new IPTObject()); + } + + void EndObject () + { + CurrentSemanticValue.o = m_value_stack.Pop(); + } + } + + internal class IPTObject + { + Hashtable m_dict = new Hashtable(); + ArrayList m_values = new ArrayList(); + + public ArrayList Values { get { return m_values; } } + + public object this[string field] + { + get { return m_dict[field]; } + set { m_dict[field] = value; } + } + + public bool Contains (string field) + { + return m_dict.ContainsKey (field); + } + } +} diff --git a/Experimental/Artemis/IPT.Scanner.Generated.cs b/Experimental/Artemis/IPT.Scanner.Generated.cs new file mode 100644 index 00000000..2f43554c --- /dev/null +++ b/Experimental/Artemis/IPT.Scanner.Generated.cs @@ -0,0 +1,713 @@ +// +// This CSharp output file generated by Gardens Point LEX +// Gardens Point LEX (GPLEX) is Copyright (c) John Gough, QUT 2006-2014. +// Output produced by GPLEX is the property of the user. +// See accompanying file GPLEXcopyright.rtf. +// +// GPLEX Version: 1.2.2 +// GPLEX frame file +// +// Option settings: verbose, parser, stack, minimize +// Option settings: compressNext, persistBuffer, noEmbedBuffers +// + +// +// Revised backup code +// Version 1.2.1 of 24-June-2013 +// +// +#define STACK +#define PERSIST +#define BYTEMODE + +using System; +using System.IO; +using System.Text; +using System.Globalization; +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Diagnostics.CodeAnalysis; + +using QUT.GplexBuffers; + +namespace GameRes.Formats.Artemis +{ + /// + /// Summary Canonical example of GPLEX automaton + /// + +#if STANDALONE + // + // These are the dummy declarations for stand-alone GPLEX applications + // normally these declarations would come from the parser. + // If you declare /noparser, or %option noparser then you get this. + // + + internal enum Token + { + EOF = 0, maxParseToken = int.MaxValue + // must have at least these two, values are almost arbitrary + } + + internal abstract class ScanBase + { + [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] + public abstract int yylex(); + + [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yywrap")] + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yywrap")] + protected virtual bool yywrap() { return true; } + +#if BABEL + protected abstract int CurrentSc { get; set; } + // EolState is the 32-bit of state data persisted at + // the end of each line for Visual Studio colorization. + // The default is to return CurrentSc. You must override + // this if you want more complicated behavior. + public virtual int EolState { + get { return CurrentSc; } + set { CurrentSc = value; } + } + } + + internal interface IColorScan + { + void SetSource(string source, int offset); + int GetNext(ref int state, out int start, out int end); +#endif // BABEL + } + +#endif // STANDALONE + + // If the compiler can't find the scanner base class maybe you + // need to run GPPG with the /gplex option, or GPLEX with /noparser +#if BABEL + internal sealed partial class IPTScanner : ScanBase, IColorScan + { + private ScanBuff buffer; + int currentScOrd; // start condition ordinal + + protected override int CurrentSc + { + // The current start state is a property + // to try to avoid the user error of setting + // scState but forgetting to update the FSA + // start state "currentStart" + // + get { return currentScOrd; } // i.e. return YY_START; + set { currentScOrd = value; // i.e. BEGIN(value); + currentStart = startState[value]; } + } +#else // BABEL + internal sealed partial class IPTScanner : ScanBase + { + private ScanBuff buffer; + int currentScOrd; // start condition ordinal +#endif // BABEL + + /// + /// The input buffer for this scanner. + /// + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + public ScanBuff Buffer { get { return buffer; } } + + private static int GetMaxParseToken() { + System.Reflection.FieldInfo f = typeof(Token).GetField("maxParseToken"); + return (f == null ? int.MaxValue : (int)f.GetValue(null)); + } + + static int parserMax = GetMaxParseToken(); + + enum Result {accept, noMatch, contextFound}; + + const int maxAccept = 8; + const int initial = 9; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + +#region user code +#endregion user code + + int state; + int currentStart = startState[0]; + int code; // last code read + int cCol; // column number of code + int lNum; // current line number + // + // The following instance variables are used, among other + // things, for constructing the yylloc location objects. + // + int tokPos; // buffer position at start of token + int tokCol; // zero-based column number at start of token + int tokLin; // line number at start of token + int tokEPos; // buffer position at end of token + int tokECol; // column number at end of token + int tokELin; // line number at end of token + string tokTxt; // lazily constructed text of token +#if STACK + private Stack scStack = new Stack(); +#endif // STACK + +#region ScannerTables + struct Table { + public int min; public int rng; public int dflt; + public sbyte[] nxt; + public Table(int m, int x, int d, sbyte[] n) { + min = m; rng = x; dflt = d; nxt = n; + } + }; + + static int[] startState = new int[] {9, 0}; + + static Table[] NxS = new Table[12] { +/* NxS[ 0] */ new Table(0, 0, 0, null), // Shortest string "" +/* NxS[ 1] */ // Shortest string "\t" + new Table(9, 24, -1, new sbyte[] {1, 1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1}), +/* NxS[ 2] */ new Table(0, 0, -1, null), // Shortest string "," +/* NxS[ 3] */ // Shortest string "0" + new Table(48, 10, -1, new sbyte[] {3, 3, 3, 3, 3, 3, + 3, 3, 3, 3}), +/* NxS[ 4] */ new Table(0, 0, -1, null), // Shortest string "=" +/* NxS[ 5] */ // Shortest string "A" + new Table(65, 58, -1, new sbyte[] {5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5}), +/* NxS[ 6] */ new Table(0, 0, -1, null), // Shortest string "{" +/* NxS[ 7] */ new Table(0, 0, -1, null), // Shortest string "}" +/* NxS[ 8] */ new Table(0, 0, -1, null), // Shortest string "\"\"" +/* NxS[ 9] */ // Shortest string "" + new Table(9, 117, -1, new sbyte[] {1, 1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, + -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -1, -1, -1, 4, -1, + -1, -1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, -1, -1, -1, -1, + -1, -1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, -1, 7}), +/* NxS[ 10] */ // Shortest string "\"" + new Table(10, 83, 10, new sbyte[] {-1, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11}), +/* NxS[ 11] */ // Shortest string "\"\\" + new Table(10, 1, 10, new sbyte[] {-1}), + }; + +int NextState() { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + unchecked { + int rslt; + int idx = (byte)(code - NxS[state].min); + if ((uint)idx >= (uint)NxS[state].rng) rslt = NxS[state].dflt; + else rslt = NxS[state].nxt[idx]; + return rslt; + } +} + +#endregion + + +#if BACKUP + // ============================================================== + // == Nested struct used for backup in automata that do backup == + // ============================================================== + + struct Context // class used for automaton backup. + { + public int bPos; + public int rPos; // scanner.readPos saved value + public int cCol; + public int lNum; // Need this in case of backup over EOL. + public int state; + public int cChr; + } + + private Context ctx = new Context(); +#endif // BACKUP + + // ============================================================== + // ==== Nested struct to support input switching in scanners ==== + // ============================================================== + + struct BufferContext { + internal ScanBuff buffSv; + internal int chrSv; + internal int cColSv; + internal int lNumSv; + } + + // ============================================================== + // ===== Private methods to save and restore buffer contexts ==== + // ============================================================== + + /// + /// This method creates a buffer context record from + /// the current buffer object, together with some + /// scanner state values. + /// + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + BufferContext MkBuffCtx() + { + BufferContext rslt; + rslt.buffSv = this.buffer; + rslt.chrSv = this.code; + rslt.cColSv = this.cCol; + rslt.lNumSv = this.lNum; + return rslt; + } + + /// + /// This method restores the buffer value and allied + /// scanner state from the given context record value. + /// + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + void RestoreBuffCtx(BufferContext value) + { + this.buffer = value.buffSv; + this.code = value.chrSv; + this.cCol = value.cColSv; + this.lNum = value.lNumSv; + } + // =================== End Nested classes ======================= + +#if !NOFILES + internal IPTScanner(Stream file) { + SetSource(file); // no unicode option + } +#endif // !NOFILES + + internal IPTScanner() { } + + private int readPos; + + void GetCode() + { + if (code == '\n') // This needs to be fixed for other conventions + // i.e. [\r\n\205\u2028\u2029] + { + cCol = -1; + lNum++; + } + readPos = buffer.Pos; + + // Now read new codepoint. + code = buffer.Read(); + if (code > ScanBuff.EndOfFile) + { +#if (!BYTEMODE) + if (code >= 0xD800 && code <= 0xDBFF) + { + int next = buffer.Read(); + if (next < 0xDC00 || next > 0xDFFF) + code = ScanBuff.UnicodeReplacementChar; + else + code = (0x10000 + ((code & 0x3FF) << 10) + (next & 0x3FF)); + } +#endif + cCol++; + } + } + + void MarkToken() + { +#if (!PERSIST) + buffer.Mark(); +#endif + tokPos = readPos; + tokLin = lNum; + tokCol = cCol; + } + + void MarkEnd() + { + tokTxt = null; + tokEPos = readPos; + tokELin = lNum; + tokECol = cCol; + } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + int Peek() + { + int rslt, codeSv = code, cColSv = cCol, lNumSv = lNum, bPosSv = buffer.Pos; + GetCode(); rslt = code; + lNum = lNumSv; cCol = cColSv; code = codeSv; buffer.Pos = bPosSv; + return rslt; + } + + // ============================================================== + // ===== Initialization of string-based input buffers ==== + // ============================================================== + + /// + /// Create and initialize a StringBuff buffer object for this scanner + /// + /// the input string + /// starting offset in the string + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + public void SetSource(string source, int offset) + { + this.buffer = ScanBuff.GetBuffer(source); + this.buffer.Pos = offset; + this.lNum = 0; + this.code = '\n'; // to initialize yyline, yycol and lineStart + GetCode(); + } + + // ================ LineBuffer Initialization =================== + /// + /// Create and initialize a LineBuff buffer object for this scanner + /// + /// the list of input strings + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + public void SetSource(IList source) + { + this.buffer = ScanBuff.GetBuffer(source); + this.code = '\n'; // to initialize yyline, yycol and lineStart + this.lNum = 0; + GetCode(); + } + +#if !NOFILES + // =============== StreamBuffer Initialization ================== + + /// + /// Create and initialize a StreamBuff buffer object for this scanner. + /// StreamBuff is buffer for 8-bit byte files. + /// + /// the input byte stream + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + public void SetSource(Stream source) + { + this.buffer = ScanBuff.GetBuffer(source); + this.lNum = 0; + this.code = '\n'; // to initialize yyline, yycol and lineStart + GetCode(); + } + +#if !BYTEMODE + // ================ TextBuffer Initialization =================== + + /// + /// Create and initialize a TextBuff buffer object for this scanner. + /// TextBuff is a buffer for encoded unicode files. + /// + /// the input text file + /// Code page to use if file has + /// no BOM. For 0, use machine default; for -1, 8-bit binary + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + public void SetSource(Stream source, int fallbackCodePage) + { + this.buffer = ScanBuff.GetBuffer(source, fallbackCodePage); + this.lNum = 0; + this.code = '\n'; // to initialize yyline, yycol and lineStart + GetCode(); + } +#endif // !BYTEMODE +#endif // !NOFILES + + // ============================================================== + +#if BABEL + // + // Get the next token for Visual Studio + // + // "state" is the inout mode variable that maintains scanner + // state between calls, using the EolState property. In principle, + // if the calls of EolState are costly set could be called once + // only per line, at the start; and get called only at the end + // of the line. This needs more infrastructure ... + // + public int GetNext(ref int state, out int start, out int end) + { + Token next; + int s, e; + s = state; // state at start + EolState = state; + next = (Token)Scan(); + state = EolState; + e = state; // state at end; + start = tokPos; + end = tokEPos - 1; // end is the index of last char. + return (int)next; + } +#endif // BABEL + + // ======== AbstractScanner<> Implementation ========= + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] + public override int yylex() + { + // parserMax is set by reflecting on the Tokens + // enumeration. If maxParseToken is defined + // that is used, otherwise int.MaxValue is used. + int next; + do { next = Scan(); } while (next >= parserMax); + return next; + } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + int yypos { get { return tokPos; } } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + int yyline { get { return tokLin; } } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + int yycol { get { return tokCol; } } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yytext")] + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yytext")] + public string yytext + { + get + { + if (tokTxt == null) + tokTxt = buffer.GetString(tokPos, tokEPos); + return tokTxt; + } + } + + /// + /// Discards all but the first "n" codepoints in the recognized pattern. + /// Resets the buffer position so that only n codepoints have been consumed; + /// yytext is also re-evaluated. + /// + /// The number of codepoints to consume + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + void yyless(int n) + { + buffer.Pos = tokPos; + // Must read at least one char, so set before start. + cCol = tokCol - 1; + GetCode(); + // Now ensure that line counting is correct. + lNum = tokLin; + // And count the rest of the text. + for (int i = 0; i < n; i++) GetCode(); + MarkEnd(); + } + + // + // It would be nice to count backward in the text + // but it does not seem possible to re-establish + // the correct column counts except by going forward. + // + /// + /// Removes the last "n" code points from the pattern. + /// + /// The number to remove + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + void _yytrunc(int n) { yyless(yyleng - n); } + + // + // This is painful, but we no longer count + // codepoints. For the overwhelming majority + // of cases the single line code is fast, for + // the others, well, at least it is all in the + // buffer so no files are touched. Note that we + // can't use (tokEPos - tokPos) because of the + // possibility of surrogate pairs in the token. + // + /// + /// The length of the pattern in codepoints (not the same as + /// string-length if the pattern contains any surrogate pairs). + /// + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyleng")] + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyleng")] + public int yyleng + { + get { + if (tokELin == tokLin) + return tokECol - tokCol; + else +#if BYTEMODE + return tokEPos - tokPos; +#else + { + int ch; + int count = 0; + int save = buffer.Pos; + buffer.Pos = tokPos; + do { + ch = buffer.Read(); + if (!char.IsHighSurrogate((char)ch)) count++; + } while (buffer.Pos < tokEPos && ch != ScanBuff.EndOfFile); + buffer.Pos = save; + return count; + } +#endif // BYTEMODE + } + } + + // ============ methods available in actions ============== + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal int YY_START { + get { return currentScOrd; } + set { currentScOrd = value; + currentStart = startState[value]; + } + } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal void BEGIN(int next) { + currentScOrd = next; + currentStart = startState[next]; + } + + // ============== The main tokenizer code ================= + + int Scan() { + for (; ; ) { + int next; // next state to enter +#if LEFTANCHORS + for (;;) { + // Discard characters that do not start any pattern. + // Must check the left anchor condition after *every* GetCode! + state = ((cCol == 0) ? anchorState[currentScOrd] : currentStart); + if ((next = NextState()) != goStart) break; // LOOP EXIT HERE... + GetCode(); + } + +#else // !LEFTANCHORS + state = currentStart; + while ((next = NextState()) == goStart) { + // At this point, the current character has no + // transition from the current state. We discard + // the "no-match" char. In traditional LEX such + // characters are echoed to the console. + GetCode(); + } +#endif // LEFTANCHORS + // At last, a valid transition ... + MarkToken(); + state = next; + GetCode(); +#if BACKUP + bool contextSaved = false; + while ((next = NextState()) > eofNum) { // Exit for goStart AND for eofNum + if (state <= maxAccept && next > maxAccept) { // need to prepare backup data + // Store data for the *latest* accept state that was found. + SaveStateAndPos( ref ctx ); + contextSaved = true; + } + state = next; + GetCode(); + } + if (state > maxAccept && contextSaved) + RestoreStateAndPos( ref ctx ); +#else // BACKUP + while ((next = NextState()) > eofNum) { // Exit for goStart AND for eofNum + state = next; + GetCode(); + } +#endif // BACKUP + if (state <= maxAccept) { + MarkEnd(); +#region ActionSwitch +#pragma warning disable 162, 1522 + switch (state) + { + case eofNum: + if (yywrap()) + return (int)Token.EOF; + break; + case 1: // Recognized '{Space}+', Shortest string "\t" +/* skip */ + break; + case 2: // Recognized '","', Shortest string "," +return ','; + break; + case 3: // Recognized '{Number}', Shortest string "0" +GetNumber(); return (int)Token.NUMBER; + break; + case 4: // Recognized '"="', Shortest string "=" +return '='; + break; + case 5: // Recognized '[a-zA-Z]+', Shortest string "A" +yylval.s = yytext; return (int)Token.IDENTIFIER; + break; + case 6: // Recognized '"{"', Shortest string "{" +return '{'; + break; + case 7: // Recognized '"}"', Shortest string "}" +return '}'; + break; + case 8: // Recognized '\"(\\.|[^\\"\n])*\"', Shortest string "\"\"" +GetStringLiteral(); return (int)Token.STRING_LITERAL; + break; + default: + break; + } +#pragma warning restore 162, 1522 +#endregion + } + } + } + +#if BACKUP + void SaveStateAndPos(ref Context ctx) { + ctx.bPos = buffer.Pos; + ctx.rPos = readPos; + ctx.cCol = cCol; + ctx.lNum = lNum; + ctx.state = state; + ctx.cChr = code; + } + + void RestoreStateAndPos(ref Context ctx) { + buffer.Pos = ctx.bPos; + readPos = ctx.rPos; + cCol = ctx.cCol; + lNum = ctx.lNum; + state = ctx.state; + code = ctx.cChr; + } +#endif // BACKUP + + // ============= End of the tokenizer code ================ + +#if STACK + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal void yy_clear_stack() { scStack.Clear(); } + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal int yy_top_state() { return scStack.Peek(); } + + internal void yy_push_state(int state) + { + scStack.Push(currentScOrd); + BEGIN(state); + } + + internal void yy_pop_state() + { + // Protect against input errors that pop too far ... + if (scStack.Count > 0) { + int newSc = scStack.Pop(); + BEGIN(newSc); + } // Otherwise leave stack unchanged. + } + #endif // STACK + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal void ECHO() { Console.Out.Write(yytext); } + + } // end class $Scanner + + +} // end namespace diff --git a/Experimental/Artemis/IPT.Scanner.cs b/Experimental/Artemis/IPT.Scanner.cs new file mode 100644 index 00000000..a5e48770 --- /dev/null +++ b/Experimental/Artemis/IPT.Scanner.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace GameRes.Formats.Artemis +{ + internal partial class IPTScanner + { + void GetNumber() + { + yylval.n = int.Parse (yytext); + yylval.s = null; + } + + void GetStringLiteral () + { + yylval.s = yytext.Substring (1, yytext.Length-2); + } + + public override void yyerror (string format, params object[] args) + { + base.yyerror (format, args); + if (args.Length > 0) + throw new YYParseException (string.Format (format, args)); + else + throw new YYParseException (format); + } + } + + public class YYParseException : Exception + { + public YYParseException (string message) : base (message) + { + } + } +} diff --git a/Experimental/Artemis/IPT.parser b/Experimental/Artemis/IPT.parser new file mode 100644 index 00000000..dabe84d5 --- /dev/null +++ b/Experimental/Artemis/IPT.parser @@ -0,0 +1,4 @@ +Parser placeholder +------------------ + +This is only a placeholder file for grouping the component files (LEX, YACC) that compose the parser. diff --git a/Experimental/Artemis/ImageIPT.cs b/Experimental/Artemis/ImageIPT.cs new file mode 100644 index 00000000..a02863a7 --- /dev/null +++ b/Experimental/Artemis/ImageIPT.cs @@ -0,0 +1,210 @@ +//! \file ImageIPT.cs +//! \date 2019 Jan 27 +//! \brief IPT composite image desciption. +// +// Copyright (C) 2019 by morkt +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.IO; +using System.Linq; +using System.Windows; +using System.Windows.Media; +using System.Windows.Media.Imaging; + +namespace GameRes.Formats.Artemis +{ + internal class IptTile + { + public int Id; + public string FileName; + public int X; + public int Y; + } + + internal class IptMetaData : ImageMetaData + { + public string Mode; + public string BaseName; + public IEnumerable Tiles; + } + + [Export(typeof(ImageFormat))] + public class IptFormat : ImageFormat + { + public override string Tag { get { return "IPT"; } } + public override string Description { get { return "Artemis composite image descriptor"; } } + public override uint Signature { get { return 0; } } + + public override ImageMetaData ReadMetaData (IBinaryStream file) + { + if (!file.Name.HasExtension (".ipt")) + return null; + + var parser = new IPTParser(); + parser.Parse (file.AsStream); + var ipt = parser.RootObject["ipt"] as IPTObject; + if (null == ipt) + return null; + var mode = ipt["mode"] as string; + var canvas = ipt["base"] as IPTObject; + if (null == mode || null == canvas) + return null; + var tiles = ipt.Values.Cast() .Select (t => new IptTile { + Id = (int)t["id"], + FileName = (string)t["file"], + X = (int)t["x"], + Y = (int)t["y"], + }); // XXX order by Id? + if ("cut" == mode && !tiles.Any()) + return null; + return new IptMetaData { + Width = (uint)(int)canvas["w"], + Height = (uint)(int)canvas["h"], + OffsetX = (int)canvas["x"], + OffsetY = (int)canvas["y"], + BPP = 32, + Mode = mode, + BaseName = (string)canvas.Values[0], + Tiles = tiles.ToList(), + }; + } + + public override ImageData Read (IBinaryStream file, ImageMetaData info) + { + var meta = (IptMetaData)info; + if (meta.Mode != "diff" && meta.Mode != "cut") + throw new InvalidFormatException (string.Format ("Not supported IPT tile mode '{0}'.", meta.Mode)); + var canvas = new WriteableBitmap (meta.iWidth, meta.iHeight, ImageData.DefaultDpiX, + ImageData.DefaultDpiY, PixelFormats.Bgr32, null); + var base_dir = VFS.GetDirectoryName (file.Name); + try + { + canvas.Lock(); + if ("diff" == meta.Mode) + { + var base_name = VFS.CombinePath (base_dir, meta.BaseName + ".png"); + ReadIntoCanvas (base_name, canvas, 0, 0); + } + foreach (var tile in meta.Tiles) + { + var tile_name = VFS.CombinePath (base_dir, tile.FileName + ".png"); + ReadIntoCanvas (tile_name, canvas, tile.X, tile.Y, true); + } + } + finally + { + canvas.Unlock(); + } + canvas.Freeze(); + return new ImageData (canvas, meta); + } + + void ReadIntoCanvas (string filename, WriteableBitmap output, int x, int y, bool blend = false) + { + if (y >= output.PixelHeight || x >= output.PixelWidth) + return; + var tile = ReadBitmapFromFile (filename); + int src_x = x >= 0 ? 0 : -x; + int src_y = y >= 0 ? 0 : -y; + var width = Math.Min (tile.PixelWidth - src_x, output.PixelWidth - x); + var height = Math.Min (tile.PixelHeight - src_y, output.PixelHeight - y); + var source = new Int32Rect (src_x, src_y, width, height); + if (!source.HasArea) + return; + if (blend && tile.Format == PixelFormats.Bgra32) + { + BlendBitmap (tile, source, output, x, y); + } + else + { + int dst_stride = output.BackBufferStride; + int offset = y * dst_stride + x * 4; + var buf_pos = output.BackBuffer + offset; + var size = output.PixelHeight * dst_stride - offset; + tile.CopyPixels (source, (IntPtr)buf_pos, size, dst_stride); + } + var target = new Int32Rect (x, y, width, height); + output.AddDirtyRect (target); + } + + void BlendBitmap (BitmapSource bitmap, Int32Rect source, WriteableBitmap output, int x, int y) + { + int src_stride = source.Width * 4; + var pixels = new byte[src_stride * source.Height]; + bitmap.CopyPixels (source, pixels, src_stride, 0); + unsafe + { + int dst_stride = output.BackBufferStride; + int offset = y * dst_stride + x * 4; + byte* buffer = (byte*)(output.BackBuffer + offset); + int src = 0; + for (int h = 0; h < source.Height; ++h) + { + int dst = 0; + for (int w = 0; w < source.Width; ++w) + { + byte src_alpha = pixels[src+3]; + if (src_alpha > 0) + { + if (0xFF == src_alpha) + { + buffer[dst ] = pixels[src]; + buffer[dst+1] = pixels[src+1]; + buffer[dst+2] = pixels[src+2]; + } + else + { + buffer[dst+0] = (byte)((pixels[src+0] * src_alpha + buffer[dst+0] * (0xFF - src_alpha)) / 0xFF); + buffer[dst+1] = (byte)((pixels[src+1] * src_alpha + buffer[dst+1] * (0xFF - src_alpha)) / 0xFF); + buffer[dst+2] = (byte)((pixels[src+2] * src_alpha + buffer[dst+2] * (0xFF - src_alpha)) / 0xFF); + } + } + dst += 4; + src += 4; + } + buffer += dst_stride; + } + } + } + + BitmapSource ReadBitmapFromFile (string filename) + { + using (var input = VFS.OpenBinaryStream (filename)) + { + var image = Read (input); + if (null == image) + throw new InvalidFormatException ("Invalid IPT tile format."); + var bitmap = image.Bitmap; + if (bitmap.Format.BitsPerPixel != 32) + bitmap = new FormatConvertedBitmap (bitmap, PixelFormats.Bgr32, null, 0); + return bitmap; + } + } + + public override void Write (Stream file, ImageData image) + { + throw new System.NotImplementedException ("IptFormat.Write not implemented"); + } + } +} diff --git a/Experimental/Artemis/ShiftReduceParserCode.cs b/Experimental/Artemis/ShiftReduceParserCode.cs new file mode 100644 index 00000000..ca2dfb94 --- /dev/null +++ b/Experimental/Artemis/ShiftReduceParserCode.cs @@ -0,0 +1,875 @@ +// Gardens Point Parser Generator +// Copyright (c) Wayne Kelly, QUT 2005-2014 +// (see accompanying GPPGcopyright.rtf) + +using System; +using System.Text; +using System.Globalization; +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Diagnostics.CodeAnalysis; + +namespace QUT.Gppg { + /// + /// Abstract class for GPPG shift-reduce parsers. + /// Parsers generated by GPPG derive from this base + /// class, overriding the abstract Initialize() and + /// DoAction() methods. + /// + /// Semantic value type + /// Location type +#if EXPORT_GPPG + public abstract class ShiftReduceParser +#else + internal abstract class ShiftReduceParser +#endif + where TSpan : IMerge, new() { + private AbstractScanner scanner; + /// + /// The abstract scanner for this parser. + /// + protected AbstractScanner Scanner { + get { return scanner; } + set { scanner = value; } + } + + /// + /// Constructor for base class + /// + /// Scanner instance for this parser + protected ShiftReduceParser( AbstractScanner scanner ) { + this.scanner = scanner; + } + + // ============================================================== + // TECHNICAL EXPLANATION. + // Why the next two fields are not exposed via properties. + // ============================================================== + // These fields are of the generic parameter types, and are + // frequently instantiated as struct types in derived classes. + // Semantic actions are defined in the derived classes and refer + // to instance fields of these structs. In such cases the code + // "get_CurrentSemanticValue().myField = blah;" will fail since + // the getter pushes the value of the field, not the reference. + // So, in the presence of properties, gppg would need to encode + // such field accesses as ... + // "tmp = get_CurrentSemanticValue(); // Fetch value + // tmp.myField = blah; // update + // set_CurrentSemanticValue(tmp); " // Write update back. + // There is no issue if TValue is restricted to be a ref type. + // The same explanation applies to scanner.yylval. + // ============================================================== + /// + /// The current value of the "$$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TValue CurrentSemanticValue; + + /// + /// The current value of the "@$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TSpan CurrentLocationSpan; + protected int NextToken; + + private TSpan LastSpan; + private State FsaState; + private bool recovering; + private int tokensSinceLastError; + + private PushdownPrefixState StateStack = new PushdownPrefixState(); + private PushdownPrefixState valueStack = new PushdownPrefixState(); + private PushdownPrefixState locationStack = new PushdownPrefixState(); + + /// + /// The stack of semantic value (YYSTYPE) values. + /// + protected PushdownPrefixState ValueStack { get { return valueStack; } } + + /// + /// The stack of location value (YYLTYPE) varlues. + /// + protected PushdownPrefixState LocationStack { get { return locationStack; } } + + private int errorToken; + private int endOfFileToken; + private string[] nonTerminals; + private State[] states; + private Rule[] rules; + + /// + /// Initialization method to allow derived classes + /// to insert the rule list into this base class. + /// + /// The array of Rule objects + protected void InitRules( Rule[] rules ) { this.rules = rules; } + + /// + /// Initialization method to allow derived classes to + /// insert the states table into this base class. + /// + /// The pre-initialized states table + protected void InitStates( State[] states ) { this.states = states; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// + /// + protected void InitStateTable( int size ) { states = new State[size]; } + + /// + /// Initialization method to allow derived classes + /// to insert the special value for the error and EOF tokens. + /// + /// The error state ordinal + /// The EOF stat ordinal + protected void InitSpecialTokens( int err, int end ) { + errorToken = err; + endOfFileToken = end; + } + + /// + /// Initialization method to allow derived classes to + /// insert the non-terminal symbol names into this base class. + /// + /// Non-terminal symbol names + protected void InitNonTerminals( string[] names ) { nonTerminals = names; } + + #region YYAbort, YYAccept etcetera. + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AcceptException : Exception { + internal AcceptException() { } + protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AbortException : Exception { + internal AbortException() { } + protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class ErrorException : Exception { + internal ErrorException() { } + protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + + // The following methods are only called from within + // a semantic action. The thrown exceptions can never + // propagate outside the ShiftReduceParser class in + // which they are nested. + + /// + /// Force parser to terminate, returning "true" + /// + protected static void YYAccept() { throw new AcceptException(); } + + /// + /// Force parser to terminate, returning "false" + /// + protected static void YYAbort() { throw new AbortException(); } + + /// + /// Force parser to terminate, returning + /// "false" if error recovery fails. + /// + protected static void YYError() { throw new ErrorException(); } + + /// + /// Check if parser in error recovery state. + /// + protected bool YYRecovering { get { return recovering; } } + #endregion + + /// + /// Abstract base method. ShiftReduceParser calls this + /// to initialize the base class data structures. Concrete + /// parser classes must override this method. + /// + protected abstract void Initialize(); + + /// + /// Main entry point of the Shift-Reduce Parser. + /// + /// True if parse succeeds, else false for + /// unrecoverable errors + public bool Parse() { + Initialize(); // allow derived classes to instantiate rules, states and nonTerminals + + NextToken = 0; + FsaState = states[0]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + + while (true) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); + DisplayStack(); +#endif + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + if (NextToken == 0) { + // We save the last token span, so that the location span + // of production right hand sides that begin or end with a + // nullable production will be correct. + LastSpan = scanner.yylloc; + NextToken = scanner.yylex(); +#if TRACE_ACTIONS + Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) ); +#endif + } +#if TRACE_ACTIONS + else + Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) ); +#endif + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + } + + if (action > 0) // shift + { + Shift( action ); + } + else if (action < 0) // reduce + { + try { + Reduce( -action ); + if (action == -1) // accept + return true; + } + catch (Exception x) { + if (x is AbortException) + return false; + else if (x is AcceptException) + return true; + else if (x is ErrorException && !ErrorRecovery()) + return false; + else + throw; // Rethrow x, preserving information. + + } + } + else if (action == 0) // error + if (!ErrorRecovery()) + return false; + } + } + + private void Shift( int stateIndex ) { +#if TRACE_ACTIONS + Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken)); +#endif + FsaState = states[stateIndex]; + + valueStack.Push( scanner.yylval ); + StateStack.Push( FsaState ); + LocationStack.Push( scanner.yylloc ); + + if (recovering) { + if (NextToken != errorToken) + tokensSinceLastError++; + + if (tokensSinceLastError > 5) + recovering = false; + } + + if (NextToken != endOfFileToken) + NextToken = 0; + } + + private void Reduce( int ruleNumber ) { +#if TRACE_ACTIONS + DisplayRule(ruleNumber); +#endif + Rule rule = rules[ruleNumber]; + int rhLen = rule.RightHandSide.Length; + // + // Default actions for unit productions. + // + if (rhLen == 1) { + CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1; + CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1; + } + else if (rhLen == 0) { + // Create a new blank value. + // Explicit semantic action may mutate this value + CurrentSemanticValue = default( TValue ); + // The location span for an empty production will start with the + // beginning of the next lexeme, and end with the finish of the + // previous lexeme. This gives the correct behaviour when this + // nonsense value is used in later Merge operations. + CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ? + scanner.yylloc.Merge( LastSpan ) : + default( TSpan )); + } + else { + // Default action: $$ = $1; + CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen]; + // Default action "@$ = @1.Merge(@N)" for location info. + TSpan at1 = LocationStack[LocationStack.Depth - rhLen]; + TSpan atN = LocationStack[LocationStack.Depth - 1]; + CurrentLocationSpan = + ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan )); + } + + DoAction( ruleNumber ); + + for (int i = 0; i < rule.RightHandSide.Length; i++) { + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + } + FsaState = StateStack.TopElement(); + + if (FsaState.Goto.ContainsKey( rule.LeftHandSide )) + FsaState = states[FsaState.Goto[rule.LeftHandSide]]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + } + + /// + /// Execute the selected action from array. + /// Must be overriden in derived classes. + /// + /// Index of the action to perform + protected abstract void DoAction( int actionNumber ); + + private bool ErrorRecovery() { + bool discard; + + if (!recovering) // if not recovering from previous error + ReportError(); + + if (!FindErrorRecoveryState()) + return false; + // + // The interim fix for the "looping in error recovery" + // artifact involved moving the setting of the recovering + // bool until after invalid tokens have been discarded. + // + ShiftErrorToken(); + discard = DiscardInvalidTokens(); + recovering = true; + tokensSinceLastError = 0; + return discard; + } + + private void ReportError() { + StringBuilder errorMsg = new StringBuilder(); + errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) ); + + if (FsaState.ParserTable.Count < 7) { + bool first = true; + foreach (int terminal in FsaState.ParserTable.Keys) { + if (first) + errorMsg.Append( ", expecting " ); + else + errorMsg.Append( ", or " ); + + errorMsg.Append( TerminalToString( terminal ) ); + first = false; + } + } + scanner.yyerror( errorMsg.ToString() ); + } + + private void ShiftErrorToken() { + int old_next = NextToken; + NextToken = errorToken; + + Shift( FsaState.ParserTable[NextToken] ); + +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); +#endif + NextToken = old_next; + } + + private bool FindErrorRecoveryState() { + while (true) // pop states until one found that accepts error token + { + if (FsaState.ParserTable != null && + FsaState.ParserTable.ContainsKey( errorToken ) && + FsaState.ParserTable[errorToken] > 0) // shift + return true; + +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number); +#endif + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + +#if TRACE_ACTIONS + DisplayStack(); +#endif + if (StateStack.IsEmpty()) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Aborting: didn't find a state that accepts error token"); +#endif + return false; + } + else + FsaState = StateStack.TopElement(); + } + } + + private bool DiscardInvalidTokens() { + + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + // Discard tokens until find one that works ... + while (true) { + if (NextToken == 0) { +#if TRACE_ACTIONS + Console.Error.Write("Reading a token: "); +#endif + NextToken = scanner.yylex(); + } +#if TRACE_ACTIONS + Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + + if (action != 0) + return true; + else { +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken)); +#endif + NextToken = 0; + } + } + } + else if (recovering && tokensSinceLastError == 0) { + // + // Boolean recovering is not set until after the first + // error token has been shifted. Thus if we get back + // here with recovering set and no tokens read we are + // looping on the same error recovery action. This + // happens if current_state.ParserTable is null because + // the state has an LR(0) reduction, but not all + // lookahead tokens are valid. This only occurs for + // error productions that *end* on "error". + // + // This action discards tokens one at a time until + // the looping stops. Another attack would be to always + // use the LALR(1) table if a production ends on "error" + // +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + NextToken = 0; + return true; + } + else + return true; + } + + /// + /// Traditional YACC method. Discards the next input token. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyclearin() { NextToken = 0; } + + /// + /// Tradional YACC method. Clear the "recovering" flag. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyerrok() { recovering = false; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// Method used by derived types to insert new + /// state instances in the "states" array. + /// + /// index of the state + /// data for the state + protected void AddState( int stateNumber, State state ) { + states[stateNumber] = state; + state.number = stateNumber; + } + + private void DisplayStack() { + Console.Error.Write( "State stack is now:" ); + for (int i = 0; i < StateStack.Depth; i++) + Console.Error.Write( " {0}", StateStack[i].number ); + Console.Error.WriteLine(); + } + + private void DisplayRule( int ruleNumber ) { + Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber ); + DisplayProduction( rules[ruleNumber] ); + } + + private void DisplayProduction( Rule rule ) { + if (rule.RightHandSide.Length == 0) + Console.Error.Write( "/* empty */ " ); + else + foreach (int symbol in rule.RightHandSide) + Console.Error.Write( "{0} ", SymbolToString( symbol ) ); + + Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) ); + } + + /// + /// Abstract state class naming terminal symbols. + /// This is overridden by derived classes with the + /// name (or alias) to be used in error messages. + /// + /// The terminal ordinal + /// + protected abstract string TerminalToString( int terminal ); + + private string SymbolToString( int symbol ) { + if (symbol < 0) + return nonTerminals[-symbol - 1]; + else + return TerminalToString( symbol ); + } + + /// + /// Return text representation of argument character + /// + /// The character to convert + /// String representation of the character + protected static string CharToString( char input ) { + switch (input) { + case '\a': return @"'\a'"; + case '\b': return @"'\b'"; + case '\f': return @"'\f'"; + case '\n': return @"'\n'"; + case '\r': return @"'\r'"; + case '\t': return @"'\t'"; + case '\v': return @"'\v'"; + case '\0': return @"'\0'"; + default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input ); + } + } + } + + /// + /// Classes implementing this interface must supply a + /// method that merges two location objects to return + /// a new object of the same type. + /// GPPG-generated parsers have the default location + /// action equivalent to "@$ = @1.Merge(@N);" where N + /// is the right-hand-side length of the production. + /// + /// The Location type +#if EXPORT_GPPG + public interface IMerge +#else + internal interface IMerge +#endif + { + /// + /// Interface method that creates a location object from + /// the current and last object. Typically used to create + /// a location object extending from the start of the @1 + /// object to the end of the @N object. + /// + /// The lexically last object to merge + /// The merged location object + TSpan Merge( TSpan last ); + } + + /// + /// This is the default class that carries location + /// information from the scanner to the parser. + /// If you don't declare "%YYLTYPE Foo" the parser + /// will expect to deal with this type. + /// +#if EXPORT_GPPG + public class LexLocation : IMerge +#else + [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )] + internal class LexLocation : IMerge +#endif + { + private int startLine; // start line + private int startColumn; // start column + private int endLine; // end line + private int endColumn; // end column + + /// + /// The line at which the text span starts. + /// + public int StartLine { get { return startLine; } } + + /// + /// The column at which the text span starts. + /// + public int StartColumn { get { return startColumn; } } + + /// + /// The line on which the text span ends. + /// + public int EndLine { get { return endLine; } } + + /// + /// The column of the first character + /// beyond the end of the text span. + /// + public int EndColumn { get { return endColumn; } } + + /// + /// Default no-arg constructor. + /// + public LexLocation() { } + + /// + /// Constructor for text-span with given start and end. + /// + /// start line + /// start column + /// end line + /// end column + public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; } + + /// + /// Create a text location which spans from the + /// start of "this" to the end of the argument "last" + /// + /// The last location in the result span + /// The merged span + public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); } + } + + /// + /// Abstract scanner class that GPPG expects its scanners to + /// extend. + /// + /// Semantic value type YYSTYPE + /// Source location type YYLTYPE +#if EXPORT_GPPG + public abstract class AbstractScanner +#else + internal abstract class AbstractScanner +#endif + where TSpan : IMerge { + /// + /// Lexical value optionally set by the scanner. The value + /// is of the %YYSTYPE type declared in the parser spec. + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + // A field must be declared for this value of parametric type, + // since it may be instantiated by a value struct. If it were + // implemented as a property, machine generated code in derived + // types would not be able to select on the returned value. +#pragma warning disable 649 + public TValue yylval; // Lexical value: set by scanner +#pragma warning restore 649 + + /// + /// Current scanner location property. The value is of the + /// type declared by %YYLTYPE in the parser specification. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual TSpan yylloc { + get { return default( TSpan ); } // Empty implementation allowing + set { /* skip */ } // yylloc to be ignored entirely. + } + + /// + /// Main call point for LEX-like scanners. Returns an int + /// corresponding to the token recognized by the scanner. + /// + /// An int corresponding to the token + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public abstract int yylex(); + + /// + /// Traditional error reporting provided by LEX-like scanners + /// to their YACC-like clients. + /// + /// Message format string + /// Optional array of args + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual void yyerror( string format, params object[] args ) { } + } + + /// + /// Encapsulated state for the parser. + /// Opaque to users, visible to the tool-generated code. + /// +#if EXPORT_GPPG + public class State + { + /// + /// The number of states in the automaton. + /// + public int number; +#else + internal class State { + /// + /// The index of this state in the states array. + /// + internal int number; +#endif + internal Dictionary ParserTable; // Terminal -> ParseAction + internal Dictionary Goto; // NonTerminal -> State; + internal int defaultAction; // = 0; // ParseAction + + /// + /// State transition data for this state. Pairs of elements of the + /// goto array associate symbol ordinals with next state indices. + /// The actions array is passed to another constructor. + /// + /// The action listc + /// Next state data + public State( int[] actions, int[] goToList ) + : this( actions ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + + /// + /// Action data for this state. Pairs of elements of the + /// action array associate action ordinals with each of + /// those symbols that have actions in the current state. + /// + /// The action array + public State( int[] actions ) { + ParserTable = new Dictionary(); + for (int i = 0; i < actions.Length; i += 2) + ParserTable.Add( actions[i], actions[i + 1] ); + } + + /// + /// Set the default action for this state. + /// + /// Ordinal of the default action + public State( int defaultAction ) { + this.defaultAction = defaultAction; + } + + /// + /// Set the default action and the state transition table. + /// + /// The default action + /// Transitions from this state + public State( int defaultAction, int[] goToList ) + : this( defaultAction ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + } + + /// + /// Rule representation at runtime. + /// +#if EXPORT_GPPG + public class Rule +#else + internal class Rule +#endif + { + internal int LeftHandSide; // symbol + internal int[] RightHandSide; // symbols + + /// + /// Rule constructor. This holds the ordinal of + /// the left hand side symbol, and the list of + /// right hand side symbols, in lexical order. + /// + /// The LHS non-terminal + /// The RHS symbols, in lexical order + public Rule( int left, int[] right ) { + this.LeftHandSide = left; + this.RightHandSide = right; + } + } + + /// + /// Stack utility for the shift-reduce parser. + /// GPPG parsers have three instances: + /// (1) The parser state stack, T = QUT.Gppg.State, + /// (2) The semantic value stack, T = TValue, + /// (3) The location stack, T = TSpan. + /// + /// +#if EXPORT_GPPG + public class PushdownPrefixState +#else + internal class PushdownPrefixState +#endif + { + // Note that we cannot use the BCL Stack class + // here as derived types need to index into stacks. + // + private T[] array = new T[8]; + private int tos = 0; + + /// + /// Indexer for values of the stack below the top. + /// + /// index of the element, starting from the bottom + /// the selected element + public T this[int index] { get { return array[index]; } } + + /// + /// The current depth of the stack. + /// + public int Depth { get { return tos; } } + + internal void Push( T value ) { + if (tos >= array.Length) { + T[] newarray = new T[array.Length * 2]; + System.Array.Copy( array, newarray, tos ); + array = newarray; + } + array[tos++] = value; + } + + internal T Pop() { + T rslt = array[--tos]; + array[tos] = default( T ); + return rslt; + } + + internal T TopElement() { return array[tos - 1]; } + + internal bool IsEmpty() { return tos == 0; } + } +} diff --git a/Experimental/Experimental.csproj b/Experimental/Experimental.csproj index b93770c5..a37bab4b 100644 --- a/Experimental/Experimental.csproj +++ b/Experimental/Experimental.csproj @@ -78,13 +78,16 @@ + + + @@ -109,11 +112,47 @@ + + + IPT.parser + + + True + True + IPT.Language.grammar.y + + + IPT.parser + + + True + True + IPT.Language.analyzer.lex + + + IPT.parser + + + IPT.parser + + + + + ipt + perl "$(SolutionDir)inc-revision.pl" "$(ProjectPath)" $(ConfigurationName) exit 0 + + $(SolutionDir)packages\YaccLexTools.0.2.2\tools\ + "$(YltTools)gplex.exe" + "$(YltTools)gppg.exe" + + + $(ProjectDir)Artemis\IPT + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. @@ -123,6 +162,18 @@ exit 0 + + + + + + + + + + + +