There is a problem with the patch committed. It incorrectly tokenizes things like
$foo = "%-{$bar}"
(this breaks the PEAR installer, amongst other things)
I've attached a fix for it.
Also, it looks like you didn't accept the part of the fix that allows for enhanced handling of heredocs. Is there a reason why? I'm sticking that in this patch again, in case you merged my last change by hand and missed that accidentally.
On Friday, November 15, 2002, at 06:48 PM, George Schlossnagle wrote:
Much sexier indeed. There are some flaws with it:
o Tokenizes heredocs on whitespace
o Doesn't count lines correctly for debug (since strings now have newlines in them)
Here's a revised patch to yours that fixes those (heredocs are tokenized on newlines - I think that is best case)
Andi Gutmans wrote:
I propose something like the following: (not tested)
It's definitely a sexier patch :)
Andi
RCS file: /repository/ZendEngine2/zend_language_scanner.l,v
retrieving revision 1.62
diff -u -u -r1.62 zend_language_scanner.l
--- zend_language_scanner.l 5 Nov 2002 22:01:35 -0000 1.62
+++ zend_language_scanner.l 15 Nov 2002 23:22:34 -0000
@@ -474,6 +474,7 @@
EXPONENT_DNUM (({LNUM}|{DNUM})[eE][+-]?{LNUM})
HNUM "0x"[0-9a-fA-F]+
LABEL [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
+ENCAPSED_STRING ([a-zA-Z0-9_\x7f-\xff \t\n\r #'.:;,()|^&+/*=%!~<>?@]|"-"[^>])+
WHITESPACE [ \n\r\t]+
TABS_AND_SPACES [ \t]*
TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@]
@@ -1076,6 +1077,12 @@
return T_VARIABLE;
}
+<ST_DOUBLE_QUOTES,ST_BACKQUOTE>{ENCAPSED_STRING} {
+ zendlval->value.str.val = (char *)estrndup(yytext, yyleng);
+ zendlval->value.str.len = yyleng;
+ zendlval->type = IS_STRING;
+ return T_STRING;
+}
<ST_IN_SCRIPTING>{LABEL} {
zendlval->value.str.val = (char *)estrndup(yytext, yyleng);
@@ -1085,7 +1092,7 @@
}
-<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{LABEL} {
+<ST_HEREDOC>{LABEL} {
zendlval->value.str.val = (char *)estrndup(yytext, yyleng);
zendlval->value.str.len = yyleng;
zendlval->type = IS_STRING;
@@ -1374,7 +1381,7 @@
}
-<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {
+<ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {
HANDLE_NEWLINES(yytext, yyleng);
zendlval->value.str.val = (char *) estrndup(yytext, yyleng);
zendlval->value.str.len = yyleng;
Andi
Index: Zend/zend_language_scanner.l
===================================================================
RCS file: /repository/Zend/zend_language_scanner.l,v
retrieving revision 1.54
diff -u -3 -r1.54 zend_language_scanner.l
--- Zend/zend_language_scanner.l 13 Nov 2002 03:28:23 -0000 1.54
+++ Zend/zend_language_scanner.l 15 Nov 2002 23:47:29 -0000
@@ -95,7 +95,7 @@
\
while (p<boundary) { \
if (*p == '\n') { \
- CG(zend_lineno)++; \
+ CG(zend_lineno)++; \
} else if ((*p == '\r') && (p+1 < boundary) && (*(p+1) != '\n')) { \
CG(zend_lineno)++; \
} \
@@ -707,6 +707,8 @@
EXPONENT_DNUM (({LNUM}|{DNUM})[eE][+-]?{LNUM})
HNUM "0x"[0-9a-fA-F]+
LABEL [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
+ENCAPSED_STRING ([a-zA-Z0-9_\x7f-\xff \t #'.:;,()|^&+/*=%!~<>?@]|"-"[^>])+
+ENCAPSED_STRING_WITH_NEWLINE ([a-zA-Z0-9_\x7f-\xff \t\n\r #'.:;,()|^&+/*=%!~<>?@]|"-"[^>])+
WHITESPACE [ \n\r\t]+
TABS_AND_SPACES [ \t]*
TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@]
@@ -1287,6 +1289,13 @@
return T_VARIABLE;
}
+<ST_DOUBLE_QUOTES,ST_BACKQUOTE>{ENCAPSED_STRING_WITH_NEWLINE} {
+ HANDLE_NEWLINES(yytext, yyleng);
+ zendlval->value.str.val = (char *)estrndup(yytext, yyleng);
+ zendlval->value.str.len = yyleng;
+ zendlval->type = IS_STRING;
+ return T_STRING;
+}
<ST_IN_SCRIPTING>{LABEL} {
zend_copy_value(zendlval, yytext, yyleng);
@@ -1295,7 +1304,7 @@
}
-<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{LABEL} {
+<ST_HEREDOC>{ENCAPSED_STRING} {
zend_copy_value(zendlval, yytext, yyleng);
zendlval->type = IS_STRING;
return T_STRING;
@@ -1598,7 +1607,7 @@
}
-<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {
+<ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {
HANDLE_NEWLINES(yytext, yyleng);
zendlval->value.str.val = (char *) estrndup(yytext, yyleng);
zendlval->value.str.len = yyleng;
--
PHP Development Mailing List <http://www.php.net/>
To unsubscribe, visit: http://www.php.net/unsub.php
-- PHP Development Mailing List <http://www.php.net/> To unsubscribe, visit: http://www.php.net/unsub.php
