Revision: 18112
Author: [email protected]
Date: Wed Nov 27 16:07:52 2013 UTC
Log: Experimental scanner: End the octal position madness.
Now we just try to have the beginning of the octal position right, and are
quite
liberal about the end. It should be fine for all practical purposes. The
error
messages will change a bit.
[email protected], [email protected]
BUG=
Review URL: https://codereview.chromium.org/91963002
http://code.google.com/p/v8/source/detail?r=18112
Modified:
/branches/experimental/parser/src/lexer/lexer-shell.cc
/branches/experimental/parser/src/lexer/lexer_py.re
=======================================
--- /branches/experimental/parser/src/lexer/lexer-shell.cc Wed Nov 27
15:28:46 2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer-shell.cc Wed Nov 27
16:07:52 2013 UTC
@@ -177,17 +177,13 @@
TokenWithLocation() :
value(Token::ILLEGAL), beg(0), end(0), is_ascii(false) { }
TokenWithLocation(Token::Value value, size_t beg, size_t end,
- int octal_beg, int octal_end) :
- value(value), beg(beg), end(end), is_ascii(false),
octal_beg(octal_beg),
- octal_end(octal_end) { }
+ int octal_beg) :
+ value(value), beg(beg), end(end), is_ascii(false),
octal_beg(octal_beg) {
+ }
bool operator==(const TokenWithLocation& other) {
- // The octal_end of the baseline scanner is inconsistent between octal
- // numbers (end = one beyond the last digit) and octal escapes (end =
the
- // last digit). Ignore that.
return value == other.value && beg == other.beg && end == other.end &&
literal == other.literal && is_ascii == other.is_ascii &&
- octal_beg == other.octal_beg &&
- octal_end >= other.octal_end - 1 && octal_end <= other.octal_end +
1;
+ octal_beg == other.octal_beg;
}
bool operator!=(const TokenWithLocation& other) {
return !(*this == other);
@@ -201,7 +197,7 @@
printf(is_ascii ? " %02x" : " %04x", literal[i]);
}
}
- printf(" (last octal: %d %d)\n", octal_beg, octal_end);
+ printf(" (last octal start: %d)\n", octal_beg);
}
};
@@ -227,8 +223,7 @@
TokenWithLocation GetTokenWithLocation(Scanner *scanner, Token::Value
token) {
int beg = scanner->location().beg_pos;
int end = scanner->location().end_pos;
- TokenWithLocation result(token, beg, end,
scanner->octal_position().beg_pos,
- scanner->octal_position().end_pos);
+ TokenWithLocation result(token, beg, end,
scanner->octal_position().beg_pos);
if (HasLiteral(token)) {
result.is_ascii = scanner->is_literal_ascii();
if (scanner->is_literal_ascii()) {
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Wed Nov 27 15:28:46
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Wed Nov 27 16:07:52
2013 UTC
@@ -198,11 +198,11 @@
default_action <do_token_and_go_forward(ILLEGAL)>
<<DoubleQuoteString>>
-"\\" line_terminator_sequence <||continue>
+"\\" line_terminator_sequence <set_has_escapes||continue>
/\\[x][:hex_digit:]{2}/ <set_has_escapes||continue>
/\\[u][:hex_digit:]{4}/ <set_has_escapes||continue>
/\\[1-7]/ <octal_inside_string||continue>
-/\\[0-7]{2,3}/ <octal_inside_string||continue>
+/\\[0-7][0-7]+/ <octal_inside_string||continue>
/\\[^xu1-7:line_terminator:]/ <set_has_escapes||continue>
"\\" <|token(ILLEGAL)|>
line_terminator <|token(ILLEGAL)|>
@@ -212,11 +212,11 @@
<<SingleQuoteString>>
# TODO subgraph for '\'
-"\\" line_terminator_sequence <||continue>
+"\\" line_terminator_sequence <set_has_escapes||continue>
/\\[x][:hex_digit:]{2}/ <set_has_escapes||continue>
/\\[u][:hex_digit:]{4}/ <set_has_escapes||continue>
/\\[1-7]/ <octal_inside_string||continue>
-/\\[0-7]{2,3}/ <octal_inside_string||continue>
+/\\[0-7][0-7]+/ <octal_inside_string||continue>
/\\[^xu1-7:line_terminator:]/ <set_has_escapes||continue>
"\\" <|token(ILLEGAL)|>
line_terminator <|token(ILLEGAL)|>
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
