Changeset: ce97d3d2c221 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ce97d3d2c221
Modified Files:
monetdb5/extras/jaql/Tests/plan01.stable.out
monetdb5/extras/jaql/jaql.c
monetdb5/extras/jaql/jaqlscenario.c
monetdb5/extras/jaql/jaqltree.h
monetdb5/extras/jaql/parser/jaql.l
monetdb5/extras/jaql/parser/jaql.y
Branch: default
Log Message:
jaql: cleanup lexer/parser for error messages
Made error messages a bit more informative by providing a snippet where
the error occurred. This required some hacking and tweaking in the
parser and lexer.
Dropped all underscores from token names for cosmetic reasons.
diffs (truncated from 571 to 300 lines):
diff --git a/monetdb5/extras/jaql/Tests/plan01.stable.out
b/monetdb5/extras/jaql/Tests/plan01.stable.out
--- a/monetdb5/extras/jaql/Tests/plan01.stable.out
+++ b/monetdb5/extras/jaql/Tests/plan01.stable.out
@@ -33,9 +33,10 @@ end main;
a as $ -> transform: { "a": $.j }
a as $ -> transform: { "a": $.j[1] }
a as $ -> transform: { "a": $.j[*].k }
-!MALException:jaql.execute:syntax error, unexpected '*', expecting _IDENT at
or around 1
+!MALException:jaql.execute:syntax error, unexpected '*', expecting IDENT at or
around '...form { "a":$.* };'
a as $ -> transform: { <to be deduced from expansion> $.* }
-!MALException:jaql.execute:syntax error, unexpected _DOT, expecting ',' or '}'
at or around 1
+!MALException:jaql.execute:syntax error, unexpected '.', expecting ',' or '}'
at or around '...ansform { $.*.foo };'
+
# 15:26:53 >
# 15:26:53 > "Done."
diff --git a/monetdb5/extras/jaql/jaql.c b/monetdb5/extras/jaql/jaql.c
--- a/monetdb5/extras/jaql/jaql.c
+++ b/monetdb5/extras/jaql/jaql.c
@@ -1675,6 +1675,7 @@ JAQLexecute(Client cntxt, MalBlkPtr mb,
j->buf = jaql;
j->err[0] = '\0';
+ j->pos = 0;
jaqllex_init_extra(j, &j->scanner);
do {
@@ -1683,7 +1684,7 @@ JAQLexecute(Client cntxt, MalBlkPtr mb,
if (j->err[0] != '\0')
break;
if (j->p == NULL)
- continue;
+ j->explain = 99; /* jump over switch below */
switch (j->explain) {
case 0: /* normal (execution) mode */
@@ -1721,11 +1722,11 @@ JAQLexecute(Client cntxt, MalBlkPtr mb,
break;
}
freetree(j->p);
- /* reset, j->buf has been reset by the lexer if EOF was found */
+ /* reset */
j->p = NULL;
j->esc_depth = 0;
j->explain = 0;
- } while (j->buf != NULL && j->err[0] == '\0');
+ } while (j->buf[j->pos + (j->tokstart - j->scanbuf)] != '\0' &&
j->err[0] == '\0');
jaqllex_destroy(j->scanner);
j->scanner = NULL;
diff --git a/monetdb5/extras/jaql/jaqlscenario.c
b/monetdb5/extras/jaql/jaqlscenario.c
--- a/monetdb5/extras/jaql/jaqlscenario.c
+++ b/monetdb5/extras/jaql/jaqlscenario.c
@@ -182,6 +182,7 @@ JAQLparser(Client c)
j->vtop = oldvtop;
j->explain = 0;
j->buf = in->buf + in->pos;
+ j->pos = 0;
jaqlparse(j);
diff --git a/monetdb5/extras/jaql/jaqltree.h b/monetdb5/extras/jaql/jaqltree.h
--- a/monetdb5/extras/jaql/jaqltree.h
+++ b/monetdb5/extras/jaql/jaqltree.h
@@ -37,6 +37,10 @@ typedef struct _jc {
int esc_depth;
char expect_json;
char *buf;
+ size_t start;
+ size_t pos;
+ char *scanbuf;
+ char *tokstart;
char err[1024];
void *scanner;
char explain;
diff --git a/monetdb5/extras/jaql/parser/jaql.l
b/monetdb5/extras/jaql/parser/jaql.l
--- a/monetdb5/extras/jaql/parser/jaql.l
+++ b/monetdb5/extras/jaql/parser/jaql.l
@@ -13,10 +13,8 @@ char *GDKstrdup(const char *);
%}
%option reentrant
+%option noyywrap
%option bison-bridge
-%option bison-locations
-%option noyywrap
-%option yylineno
%option case-insensitive
%option batch
%option nostdinit
@@ -26,10 +24,10 @@ char *GDKstrdup(const char *);
%{
YYSTYPE yylval;
-extern void jaqlerror(YYLTYPE* locp, jc* j, char const *msg);
+extern void jaqlerror(jc* j, char const *msg);
/* set line numbers each time a token is recognised */
-#define YY_USER_ACTION yylloc->first_line = yylineno;
+#define YY_USER_ACTION yyextra->tokstart = yytext;
#define YY_INPUT(buf, res, max_size) readinput(yyextra, buf, &res, max_size)
#define YY_NO_INPUT
@@ -45,20 +43,22 @@ extern void jaqlerror(YYLTYPE* locp, jc*
#define stdout (FILE *)0
static void readinput(jc *j, char *buf, int *res, size_t max_size) {
- if (j->buf != NULL && *j->buf != '\0') {
- *res = (int)strlen(j->buf);
+ if (j->buf[j->pos] != '\0') {
+ *res = (int)strlen(j->buf + j->pos);
if ((size_t)*res > max_size)
*res = (int)max_size;
- memcpy(buf, j->buf, *res);
- j->buf += *res;
+ memcpy(buf, j->buf + j->pos, *res);
+ j->start = j->pos;
+ j->pos += *res;
+ j->scanbuf = buf;
} else {
*res = YY_NULL;
}
}
%}
- /* allow to disregard JSON bits */
-%x ARR OBJ
+ /* allow to disregard JSON bits and deal with trailing garbage after ; */
+%x ARR OBJ SCOLON
%%
/* if the parser expects a JSON bit, give it */
@@ -98,27 +98,26 @@ planf return PLANF;
debug return DEBUG;
/* language constructs */
-"->" return _ARROW;
--?[0-9]+ {yylval->j_number = atol(yytext); return _NUMBER;}
--?[0-9]+\.[0-9e+-]+ {yylval->j_double = atof(yytext); return _DOUBLE;}
+"->" return ARROW;
+-?[0-9]+ {yylval->j_number = atol(yytext); return NUMBER;}
+-?[0-9]+\.[0-9e+-]+ {yylval->j_double = atof(yytext); return DOUBLE;}
\"[^\"]*\" {yytext[yyleng - 1] = '\0';
yylval->j_string = GDKstrdup(yytext + 1);
- return _STRING;}
-\$ return _DOLLAR;
-= return _ASSIGN;
-== return _EQUALS;
-!= return _NEQUAL;
-">" return _GREATER;
-">=" return _GEQUAL;
-"<" return _LESS;
-"<=" return _LEQUAL;
-not return _NOT;
-and return _AND;
-or return _OR;
-true return _TRUE;
-false return _FALSE;
-; return _SCOLON;
-\. return _DOT;
+ return STRING;}
+"$" return '$';
+"=" return ASSIGN;
+"==" return EQUALS;
+"!=" return NEQUAL;
+">" return GREATER;
+">=" return GEQUAL;
+"<" return LESS;
+"<=" return LEQUAL;
+not return NOT;
+and return AND;
+or return OR;
+true return TRUE;
+false return FALSE;
+"." return '.';
":" return ':';
"," return ',';
"(" return '(';
@@ -132,7 +131,7 @@ false return _FALSE;
if (--yyextra->esc_depth == 0) {
yylval->j_json = GDKstrdup(yytext - 1);
BEGIN(INITIAL);
- return _ARRAY;
+ return ARRAY;
} else {
yymore();
}
@@ -147,7 +146,7 @@ false return _FALSE;
if (--yyextra->esc_depth == 0) {
yylval->j_json = GDKstrdup(yytext - 1);
BEGIN(INITIAL);
- return _OBJECT;
+ return OBJECT;
} else {
yymore();
}
@@ -157,17 +156,26 @@ false return _FALSE;
}
"}" return '}';
"{" return '{';
+";" BEGIN(SCOLON);
+<SCOLON>{
+ ";" /* ignore superfluous semi-colons */;
+ [ \t\r\n]+ /* ignore whitespace */;
+ ("#"|"//").*\n /* ignore comments */;
+ <<EOF>> { BEGIN(INITIAL); return ';'; }
+ . { unput(yytext[0]); BEGIN(INITIAL); return ';'; }
+}
[ \t\r\n]+ /* ignore whitespace */;
("#"|"//").*\n /* ignore comments */;
-[a-zA-Z_][a-zA-Z0-9_]* {yylval->j_ident = GDKstrdup(yytext); return _IDENT;}
-<<EOF>> {yyextra->buf = NULL; return EOF;}
-. {
+[a-zA-Z_][a-zA-Z0-9_]* {yylval->j_ident = GDKstrdup(yytext); return IDENT;}
+<<EOF>> return EOF;
+. {
char buf[32];
snprintf(buf, sizeof(buf), "unexpected character: %c", yytext[0]);
- jaqlerror(yylloc_param, yyextra, buf);
- yyextra->buf = NULL;
+ jaqlerror(yyextra, buf);
+ yyextra->pos = yyextra->start = strlen(yyextra->buf);
+ yyextra->scanbuf = yyextra->tokstart = NULL;
YY_FLUSH_BUFFER;
BEGIN(0);
return EOF;
- }
+}
%%
diff --git a/monetdb5/extras/jaql/parser/jaql.y
b/monetdb5/extras/jaql/parser/jaql.y
--- a/monetdb5/extras/jaql/parser/jaql.y
+++ b/monetdb5/extras/jaql/parser/jaql.y
@@ -6,15 +6,15 @@
#include "jaqltree.h"
#ifdef _MSC_VER
#define snprintf _snprintf
-__declspec(dllimport)
+#define jaql_import __declspec(dllimport)
#else
-extern
+#define jaql_import extern
#endif
-char *GDKstrdup(const char *);
+jaql_import char *GDKstrdup(const char *);
+jaql_import void GDKfree(const char *);
%}
%define api.pure
-%locations
%defines
%error-verbose
@@ -30,23 +30,27 @@ char *GDKstrdup(const char *);
void *j_tree;
}
+%destructor { GDKfree($$); } <j_string>;
+%destructor { GDKfree($$); } <j_json>;
+%destructor { GDKfree($$); } <j_ident>;
+%destructor { freetree($$); } <j_tree>;
%start stmt
%token EACH FILTER TRANSFORM EXPAND GROUP INTO BY AS JOIN WHERE IN
%token SORT TOP DESC ASC EXPLAIN PLAN PLANF DEBUG UNROLL PRESERVE
-%token _ARROW _DOLLAR _ASSIGN _EQUALS _NEQUAL _TRUE _FALSE
-%token _GREATER _GEQUAL _LESS _LEQUAL _NOT _AND _OR _SCOLON _DOT
+%token ARROW ASSIGN EQUALS NEQUAL TRUE FALSE
+%token GREATER GEQUAL LESS LEQUAL NOT AND OR
-%token <j_ident> _IDENT
-%token <j_json> _ARRAY _OBJECT
+%token <j_ident> IDENT
+%token <j_json> ARRAY OBJECT
-%token <j_number> _NUMBER
-%token <j_double> _DOUBLE
-%token <j_string> _STRING
+%token <j_number> NUMBER
+%token <j_double> DOUBLE
+%token <j_string> STRING
-%type <j_tree> stmt jaql jaqlpipe opt_actions actions action predicates
+%type <j_tree> jaql jaqlpipe opt_actions actions action predicates
predicate variable and_or opt_not comparison value literal
opt_each opt_command sort_arg arith_op val_var_arith
obj_list arr_list obj_pair json_value join_var_refs join_var_ref
@@ -55,77 +59,79 @@ char *GDKstrdup(const char *);
%type <j_ident> ident
%type <j_number> opt_asc_desc opt_preserve
-/* get it right:
-http://www.cs.man.ac.uk/~pjj/cs211/ho/node8.html
-http://www.phpcompiler.org/articles/reentrantparser.html
-http://www.usualcoding.eu/post/2007/09/03/Building-a-reentrant-parser-in-C-with-Flex/Bison
-*/
%{
#define YYLEX_PARAM j->scanner
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
