diff options
Diffstat (limited to 'mcs/ilasm/scanner/ILTokenizer.cs')
| -rw-r--r-- | mcs/ilasm/scanner/ILTokenizer.cs | 371 |
1 files changed, 371 insertions, 0 deletions
diff --git a/mcs/ilasm/scanner/ILTokenizer.cs b/mcs/ilasm/scanner/ILTokenizer.cs new file mode 100644 index 00000000000..0d2730d6cee --- /dev/null +++ b/mcs/ilasm/scanner/ILTokenizer.cs @@ -0,0 +1,371 @@ +// ILTokenizer.cs
+// Author: Sergey Chaban (serge@wildwestsoftware.com)
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections;
+using System.Globalization;
+
+namespace Mono.ILASM {
+
+ public delegate void NewTokenEvent (object sender, NewTokenEventArgs args);
+
+ public class NewTokenEventArgs : EventArgs {
+
+ public readonly ILToken Token;
+
+ public NewTokenEventArgs (ILToken token)
+ {
+ Token = token;
+ }
+ }
+
+ /// <summary>
+ /// </summary>
+ public class ILTokenizer : ITokenStream {
+
+ private static readonly string idchars = "_$@?.`";
+
+ private static Hashtable keywords;
+ private static Hashtable directives;
+
+ private ILToken lastToken;
+ private ILReader reader;
+ private StringHelper strBuilder;
+ private NumberHelper numBuilder;
+ private bool in_byte_array;
+
+ public event NewTokenEvent NewTokenEvent;
+
+ static ILTokenizer()
+ {
+ keywords = ILTables.Keywords;
+ directives = ILTables.Directives;
+ }
+
+ /// <summary>
+ /// </summary>
+ /// <param name="reader"></param>
+ public ILTokenizer (StreamReader reader)
+ {
+ this.reader = new ILReader (reader);
+ strBuilder = new StringHelper (this);
+ numBuilder = new NumberHelper (this);
+ lastToken = ILToken.Invalid.Clone () as ILToken;
+ }
+
+ public ILReader Reader {
+ get {
+ return reader;
+ }
+ }
+
+ public Location Location {
+ get {
+ return reader.Location;
+ }
+ }
+
+ public bool InByteArray {
+ get { return in_byte_array; }
+ set { in_byte_array = value; }
+ }
+
+ public ILToken GetNextToken ()
+ {
+ if (lastToken == ILToken.EOF) return ILToken.EOF;
+
+ int ch;
+ int next;
+ ILToken res = ILToken.EOF.Clone () as ILToken;
+
+
+ while ((ch = reader.Read ()) != -1) {
+
+ // Comments
+ if (ch == '/') {
+ next = reader.Peek ();
+ if (next == '/') {
+ // double-slash comment, skip to the end of the line.
+ for (reader.Read ();
+ next != -1 && next != '\n';
+ next = reader.Read ());
+ continue;
+ } else if (next == '*') {
+ reader.Read ();
+ for (next = reader.Read (); next != -1; next = reader.Read ()) {
+ if (next == '*' && reader.Peek () == '/') {
+ reader.Read ();
+ goto end;
+ }
+ }
+ end:
+ continue;
+ }
+ }
+
+ // HEXBYTES are flagged by the parser otherwise it is
+ // impossible to figure them out
+ if (in_byte_array) {
+ string hx = String.Empty;
+
+ if (Char.IsWhiteSpace ((char) ch))
+ continue;
+
+ if (ch == ')') {
+ res = ILToken.CloseParens;
+ break;
+ }
+
+ if (!is_hex (ch))
+ throw new ILTokenizingException (reader.Location, ((char) ch).ToString ());
+ hx += (char) ch;
+ if (is_hex (reader.Peek ()))
+ hx += (char) reader.Read ();
+ else if (!Char.IsWhiteSpace ((char) reader.Peek ()) && reader.Peek () != ')')
+ throw new ILTokenizingException (reader.Location,
+ ((char) reader.Peek ()).ToString ());
+ res.token = Token.HEXBYTE;
+ res.val = Byte.Parse (hx, NumberStyles.HexNumber);
+
+ while (Char.IsWhiteSpace ((char) reader.Peek ()))
+ reader.Read ();
+ break;
+ }
+
+ // Ellipsis
+ if (ch == '.' && reader.Peek () == '.') {
+ reader.MarkLocation ();
+ int ch2 = reader.Read ();
+ if (reader.Peek () == '.') {
+ res = ILToken.Ellipsis;
+ reader.Read ();
+ break;
+ }
+ reader.Unread (ch2);
+ reader.RestoreLocation ();
+ }
+
+ if (ch == '.' || ch == '#') {
+ next = reader.Peek ();
+ if (ch == '.' && Char.IsDigit((char) next)) {
+ numBuilder.Start (ch);
+ reader.Unread (ch);
+ numBuilder.Build ();
+ if (numBuilder.ResultToken != ILToken.Invalid) {
+ res.CopyFrom (numBuilder.ResultToken);
+ break;
+ }
+ } else {
+ if (strBuilder.Start (next) && strBuilder.TokenId == Token.ID) {
+ reader.MarkLocation ();
+ string dirBody = strBuilder.Build ();
+ string dir = new string ((char) ch, 1) + dirBody;
+ if (IsDirective (dir)) {
+ res = ILTables.Directives [dir] as ILToken;
+ } else {
+ reader.Unread (dirBody.ToCharArray ());
+ reader.RestoreLocation ();
+ res = ILToken.Dot;
+ }
+ } else {
+ res = ILToken.Dot;
+ }
+ break;
+ }
+ }
+
+ // Numbers && Hexbytes
+ if (numBuilder.Start (ch)) {
+ if ((ch == '-') && !(Char.IsDigit ((char) reader.Peek ()))) {
+ res = ILToken.Dash;
+ break;
+ } else {
+ reader.Unread (ch);
+ numBuilder.Build ();
+ if (numBuilder.ResultToken != ILToken.Invalid) {
+ res.CopyFrom (numBuilder.ResultToken);
+ break;
+ }
+ }
+ }
+
+ // Punctuation
+ ILToken punct = ILToken.GetPunctuation (ch);
+ if (punct != null) {
+ if (punct == ILToken.Colon && reader.Peek () == ':') {
+ reader.Read ();
+ res = ILToken.DoubleColon;
+ } else {
+ res = punct;
+ }
+ break;
+ }
+
+ // ID | QSTRING | SQSTRING | INSTR_* | KEYWORD
+ if (strBuilder.Start (ch)) {
+ reader.Unread (ch);
+ string val = strBuilder.Build ();
+ if (strBuilder.TokenId == Token.ID) {
+ ILToken opcode;
+ next = reader.Peek ();
+ if (next == '.') {
+ reader.MarkLocation ();
+ reader.Read ();
+ next = reader.Peek ();
+ if (IsIdChar ((char) next)) {
+ string opTail = BuildId ();
+ string full_str = String.Format ("{0}.{1}", val, opTail);
+ opcode = InstrTable.GetToken (full_str);
+
+ if (opcode == null) {
+ if (strBuilder.TokenId != Token.ID) {
+ reader.Unread (opTail.ToCharArray ());
+ reader.Unread ('.');
+ reader.RestoreLocation ();
+ res.val = val;
+ } else {
+ res.token = Token.COMP_NAME;
+ res.val = full_str;
+ }
+ break;
+ } else {
+ res = opcode;
+ break;
+ }
+
+ } else if (Char.IsWhiteSpace ((char) next)) {
+ // Handle 'tail.' and 'unaligned.'
+ opcode = InstrTable.GetToken (val + ".");
+ if (opcode != null) {
+ res = opcode;
+ break;
+ }
+ // Let the parser handle the dot
+ reader.Unread ('.');
+ }
+ }
+ opcode = InstrTable.GetToken (val);
+ if (opcode != null) {
+ res = opcode;
+ break;
+ }
+ if (IsKeyword (val)) {
+ res = ILTables.Keywords [val] as ILToken;
+ break;
+ }
+ }
+
+ res.token = strBuilder.TokenId;
+ res.val = val;
+ break;
+ }
+ }
+
+ OnNewToken (res);
+ lastToken.CopyFrom (res);
+ return res;
+ }
+
+
+ /// <summary>
+ /// </summary>
+ public ILToken NextToken {
+ get {
+ return GetNextToken ();
+ }
+ }
+
+
+ /// <summary>
+ /// </summary>
+ public ILToken LastToken {
+ get {
+ return lastToken;
+ }
+ }
+
+ bool is_hex (int e)
+ {
+ return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
+ }
+
+ private static bool IsIdStartChar (char ch)
+ {
+ return (Char.IsLetter (ch) || (idchars.IndexOf (ch) != -1));
+ }
+
+
+ private static bool IsIdChar (char ch)
+ {
+ return (Char.IsLetterOrDigit (ch) || (idchars.IndexOf (ch) != -1));
+ }
+
+ /// <summary>
+ /// </summary>
+ /// <param name="name"></param>
+ /// <returns></returns>
+ public static bool IsOpcode (string name)
+ {
+ return InstrTable.IsInstr (name);
+ }
+
+
+ /// <summary>
+ /// </summary>
+ /// <param name="name"></param>
+ /// <returns></returns>
+ public static bool IsDirective (string name)
+ {
+ char ch = name [0];
+ bool res = (ch == '.' || ch == '#');
+
+ if (res) {
+ res = directives.Contains (name);
+ }
+
+ return res;
+ }
+
+ private string BuildId ()
+ {
+ StringBuilder idsb = new StringBuilder ();
+ int ch, last;
+
+ last = -1;
+ while ((ch = reader.Read ()) != -1) {
+ if (IsIdChar ((char) ch) || ch == '.') {
+ idsb.Append ((char) ch);
+ } else {
+ reader.Unread (ch);
+ // Never end an id on a DOT
+ if (last == '.') {
+ reader.Unread (last);
+ idsb.Length -= 1;
+ }
+ break;
+ }
+ last = ch;
+ }
+
+ return idsb.ToString ();
+ }
+
+ /// <summary>
+ /// </summary>
+ /// <param name="name"></param>
+ /// <returns></returns>
+ public static bool IsKeyword (string name)
+ {
+ return keywords.Contains (name);
+ }
+
+ private void OnNewToken (ILToken token)
+ {
+ if (NewTokenEvent != null)
+ NewTokenEvent (this, new NewTokenEventArgs (token));
+ }
+
+ }
+}
|