Author: olivier
Date: 2007-06-22 05:34:07 -0400 (Fri, 22 Jun 2007)
New Revision: 80533
Modified:
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/ChangeLog
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Parser.cs
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Tokenizer.cs
Log:
2007-06-22 Olivier Dufour <[EMAIL PROTECTED]>
* Parser.cs : badToken, labelStatement and octalStatement management
* Tokenizer.cs: bad token, hex/octal integer token management
Modified:
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/ChangeLog
===================================================================
---
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/ChangeLog
2007-06-22 08:03:37 UTC (rev 80532)
+++
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/ChangeLog
2007-06-22 09:34:07 UTC (rev 80533)
@@ -1,7 +1,13 @@
2007-06-22 Olivier Dufour <[EMAIL PROTECTED]>
- * Parser.cs : fix all unit tested thing
+ * Parser.cs : badToken, labelStatement and octalStatement management
+ * Tokenizer.cs: bad token, hex/octal integer token management
+2007-06-22 Olivier Dufour <[EMAIL PROTECTED]>
+
+ * Parser.cs : fix all unit tested thing ==> all my test is green now ;)
+
+
2007-06-17 Olivier Dufour <[EMAIL PROTECTED]>
* Parser.cs
Modified:
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Parser.cs
===================================================================
---
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Parser.cs
2007-06-22 08:03:37 UTC (rev 80532)
+++
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Parser.cs
2007-06-22 09:34:07 UTC (rev 80533)
@@ -157,15 +157,34 @@
return ParseReturn ();
case Token.Type.function:
return ParseFunctionDeclaration ();
+ case Token.Type.Semicolon:
+ return new Statement
(Statement.Operation.Empty, new TextSpan (current, current));
case Token.Type.Identifier:
- return ParseExpressionStatement ();
+ return ParseLabelStatement();
default:
- SyntaxError.Add("Statement start with a
strange token :" + Enum.GetName(typeof(Token.Type), current.Kind));
+ if ( current.Kind !=
Token.Type.LeftBrace
+ && current.Kind !=
Token.Type.Comma
+ && current.Kind !=
Token.Type.function)
+ return ParseExpressionStatement
();
break;
}
- return new Statement(Statement.Operation.Block, new
TextSpan(start,current));
+ SyntaxError.Add ("Statement start with a strange token
:" + Enum.GetName (typeof (Token.Type), current.Kind));
+ return new Statement (Statement.Operation.SyntaxError,
new TextSpan (current, current));
}
+ private LabelStatement ParseLabelStatement ()
+ {
+ Token start = current;
+ CheckSyntaxExpected (Token.Type.Identifier);
+ Identifier label = ((IdentifierToken)current).Spelling;
+ Next ();
+ CheckSyntaxExpected (Token.Type.Colon);
+ TextPoint colon = new TextPoint (current.StartPosition);
+ Next ();
+ Statement labeled = ParseStatement();
+ return new LabelStatement (label, labeled, new TextSpan
(start, current), colon);
+ }
+
private ExpressionStatement ParseExpressionStatement ()
{
Token start = current;
@@ -550,6 +569,9 @@
case Token.Type.NumericLiteral:
expr = new NumericLiteralExpression
(((NumericLiteralToken)current).Spelling, new TextSpan (current, current));
break;
+ case Token.Type.OctalIntegerLiteral:
+ expr = new OctalLiteralExpression
(((OctalIntegerLiteralToken)current).Value, new TextSpan (current, current));
+ break;
case Token.Type.HexIntegerLiteral:
expr = new HexLiteralExpression
(((HexIntegerLiteralToken)current).Value, new TextSpan (current, current));
break;
@@ -1000,12 +1022,6 @@
return ParseAssignmentExpression (noIn);
}
- /*
- LabelStatement.cs
- OctalLiteralExpression.cs
- RegularExpressionLiteralExpression.cs
- */
-
private FunctionDefinition ParseFunctionDefinition ()
{
Token start = current;
@@ -1147,10 +1163,15 @@
get { return diagnostics; }
}
- private void Error(DiagnosticCode code , TextSpan loc)
+ private void Error (DiagnosticCode code , TextSpan loc)
{
+ if (current.Kind == Token.Type.Bad)
+ code = ((BadToken)current).Diagnostic;
diagnostics.Add (new Diagnostic (code, loc));
}
+ //TODO label management (break, continue,...)
+ //TODO finished bad token and syntax error management
+ //TODO
/* TODO
BadDivideOrRegularExpressionLiteral,
EnclosingLabelShadowed,
@@ -1159,8 +1180,6 @@
ContinueContextInvalid,
ContinueLabelInvalid,
MalformedEscapeSequence,
- HexLiteralNoDigits,
- MalformedNumericLiteral,
NumericLiteralThenIdentifier,
UnterminatedStringLiteral,
UnterminatedComment,
Modified:
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Tokenizer.cs
===================================================================
---
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Tokenizer.cs
2007-06-22 08:03:37 UTC (rev 80532)
+++
trunk/olive/class/Microsoft.JScript.Compiler/Microsoft.JScript.Compiler/Tokenizer.cs
2007-06-22 09:34:07 UTC (rev 80533)
@@ -29,7 +29,7 @@
// move that to static ctor?
InitKeywords();
}
- //TODO : unicode \u
+
public Token GetNext()
{
//Must use CreateToken everytime we create a Token to
avoid issue of location
@@ -281,9 +281,14 @@
case '_':
return this.CreateIdentOrKeyword (c);
}
- return CreateToken (Token.Type.Bad);
+ return CreateBadToken
(((char)c).ToString(),DiagnosticCode.ExtraneousCharacter);
}
+ private Token CreateBadToken (string spelling, DiagnosticCode
code)
+ {
+ return new BadToken (spelling, code, position, Line,
Column, FirstOnLine);
+ }
+
public RegularExpressionLiteralToken ScanRegularExpression
(Token Divide)
{
int startpos = Divide.StartPosition;
@@ -322,6 +327,7 @@
#region token
+
private Token CreateToken (Token.Type tokenType)
{
current = new Token (tokenType, position, Line, Column,
FirstOnLine);
@@ -347,7 +353,7 @@
if (c == quote)
break;
if (c == -1 || c == '\n') {
- return new BadToken ("string is never
closed!", DiagnosticCode.SyntaxError, startPosition, startLine, startColumn,
FirstOnLine);
+ return CreateBadToken
(builder.ToString(), DiagnosticCode.SyntaxError);
}
if (c == '\\') {
ReadChar ();
@@ -393,7 +399,7 @@
else if (d >=
'a' && d <= 'f')
d = d -
'a' + 10;
else
- return
new BadToken ("", DiagnosticCode.HexLiteralNoDigits, startPosition, startLine,
startColumn, FirstOnLine);
+ return
CreateBadToken (builder.ToString(), DiagnosticCode.HexLiteralNoDigits);
r2 = (r2 << 4)
| d;
}
c = r2;
@@ -411,14 +417,14 @@
else if (d >=
'a' && d <= 'f')
d = d -
'a' + 10;
else
- return
new BadToken ("", DiagnosticCode.HexLiteralNoDigits, startPosition, startLine,
startColumn, FirstOnLine);
+ return
CreateBadToken (builder.ToString(), DiagnosticCode.HexLiteralNoDigits);
r4 = (r4 << 4)
| d;
}
c = r4;
break;
default:
//TODO 0 [lookahead
?DecimalDigit] or NonEscapeCharacter
- return new BadToken
("", DiagnosticCode.HexLiteralNoDigits, startPosition, startLine, startColumn,
FirstOnLine);
+ return CreateBadToken
(builder.ToString(), DiagnosticCode.HexLiteralNoDigits);
}
}
builder.Append ((char)c);
@@ -465,12 +471,33 @@
private Token CreateNumericLiteralToken (int first)
{
+
+ int next = PeekChar ();
+
+ if (first == '0') {
+ if (next == 'x' || next == 'X')
+ return CreateHexIntegerLiteralToken
(first);
+ else {
+ switch (next) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ return
CreateOctalIntegerLiteralToken (first);
+ }//else continue to make a numerical
literal token
+ }
+ }
+
StringBuilder builder = new StringBuilder ();
builder.Append ((char) first);
int dot = 0;
while (Advance ()) {
- int next = PeekChar ();
+ next = PeekChar ();
switch (next) {
case '0':
case '1':
@@ -489,11 +516,11 @@
case '.': {
ReadChar ();
dot++;
+ builder.Append
((char)next);
if (dot > 1) {
- current = new
Token (Token.Type.Bad, position, Line, Column, FirstOnLine);
+ current =
CreateBadToken (builder.ToString(), DiagnosticCode.MalformedNumericLiteral);
return current;
- }
- builder.Append ((char)
next);
+ }
continue;
}
}
@@ -504,6 +531,91 @@
return current;
}
+ private Token CreateOctalIntegerLiteralToken (int first)
+ {
+ StringBuilder builder = new StringBuilder ();
+ builder.Append ((char)first);
+ double val = 0;
+
+ while (Advance ()) {
+ int next = PeekChar ();
+ switch (next) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7': {
+ builder.Append
((char)next);
+ val = val * 8 + next -
'0';
+ ReadChar ();
+ continue;
+ }
+ }
+ break;
+ }
+ string result = builder.ToString ();
+
+ current = new OctalIntegerLiteralToken (result, val
,position, Line, Column, FirstOnLine);
+ return current;
+ }
+
+ private Token CreateHexIntegerLiteralToken (int first)
+ {
+ StringBuilder builder = new StringBuilder ();
+ builder.Append ((char)first);
+ double val = 0;
+ while (Advance ()) {
+ int next = PeekChar ();
+ switch (next) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': {
+ builder.Append
((char)next);
+ val = val * 16 + next -
'0';
+ ReadChar ();
+ continue;
+ }
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f': {
+ builder.Append
((char)next);
+ val = val * 16 + next -
'a' + 10;
+ ReadChar ();
+ continue;
+ }
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F': {
+ builder.Append
((char)next);
+ val = val * 16 + next -
'A' + 10;
+ ReadChar ();
+ continue;
+ }
+ }
+ break;
+ }
+ string result = builder.ToString ();
+
+ current = new HexIntegerLiteralToken (result, val,
position, Line, Column, FirstOnLine);
+ return current;
+ }
+
#endregion
private void InitKeywords ()
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches
