Revision: 19989
Author: [email protected]
Date: Mon Mar 17 13:02:24 2014 UTC
Log: Experimental parser: more correct utf8 handling
[email protected]
BUG=
Review URL: https://codereview.chromium.org/201693003
http://code.google.com/p/v8/source/detail?r=19989
Modified:
/branches/experimental/parser/src/lexer/lexer.cc
=======================================
--- /branches/experimental/parser/src/lexer/lexer.cc Mon Mar 17 11:36:36
2014 UTC
+++ /branches/experimental/parser/src/lexer/lexer.cc Mon Mar 17 13:02:24
2014 UTC
@@ -162,6 +162,30 @@
Scan();
return current_.token;
}
+
+
+static uint32_t Advance(const int8_t** buffer, const int8_t* end) {
+ unsigned bytes_read = 0;
+ uint32_t c = unibrow::Utf8::ValueOf(reinterpret_cast<const
uint8_t*>(*buffer),
+ end - *buffer,
+ &bytes_read);
+ *buffer += bytes_read;
+ return c;
+}
+
+
+static inline uint32_t Advance(const uint8_t** buffer, const uint8_t* end)
{
+ uint32_t c = **buffer;
+ (*buffer)++;
+ return c;
+}
+
+
+static inline uint32_t Advance(const uint16_t** buffer, const uint16_t*
end) {
+ uint32_t c = **buffer;
+ (*buffer)++;
+ return c;
+}
template<typename Char>
@@ -654,7 +678,7 @@
if (token.has_escapes) {
for (const Char* cursor = start; cursor != end;) {
if (*cursor != '\\') {
- literal->buffer.AddChar(*cursor++);
+ literal->buffer.AddChar(Advance(&cursor, end));
} else if (token.token == Token::IDENTIFIER) {
uc32 c;
cursor = ScanIdentifierUnicodeEscape(cursor, end, &c);
@@ -668,10 +692,8 @@
}
}
} else {
- // TODO(dcarney): This can only happen for utf8 strings
- // use a helper function.
for (const Char* cursor = start; cursor != end;) {
- literal->buffer.AddChar(*cursor++);
+ literal->buffer.AddChar(Advance(&cursor, end));
}
}
literal->SetStringFromLiteralBuffer();
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
