Paolo Bonzini <[email protected]> writes:
> In order to avoid stashing all the tokens corresponding to a JSON value,
> embed the parsing stack and state machine in JSONParser. This is more
> efficient and allows for more prompt error recovery; it also does not
> make the code substantially larger than the current recursive descent
> parser, though the state machine is probably a bit harder to follow.
>
> The stack consists of QLists and QDicts corresponding to open
> brackets and braces, plus optionally a QString with the current
> key on top of each QDict.
>
> After each value is parsed, it is added to the top array or dictionary
> or, if the stack is empty, json_parser_feed returns the complete
> QObject.
>
> For now, json-streamer.c keeps tracking the tokens up until braces
> and brackets are balanced, and then shoves the whole queue of tokens
> into the push parser. The only logic change is that JSON_END_OF_INPUT
> always triggers the emptying of the queue; the parser takes notice and
> checks that there is nothing on the stack. Not using brace_count
> and bracket_count for this is the first step towards improved separation
> of concerns between json-parser.c and json-streamer.c.
>
> Signed-off-by: Paolo Bonzini <[email protected]>
> ---
> include/qobject/json-parser.h | 6 +
> qobject/json-parser-int.h | 5 +-
> qobject/json-parser.c | 551 ++++++++++++++++++++--------------
> qobject/json-streamer.c | 21 +-
> 4 files changed, 345 insertions(+), 238 deletions(-)
>
> diff --git a/include/qobject/json-parser.h b/include/qobject/json-parser.h
> index 7345a9bd5cb..05346fa816b 100644
> --- a/include/qobject/json-parser.h
> +++ b/include/qobject/json-parser.h
> @@ -20,6 +20,12 @@ typedef struct JSONLexer {
> int x, y;
> } JSONLexer;
>
> +typedef struct JSONParserContext {
> + Error *err;
> + GQueue *stack;
> + va_list *ap;
> +} JSONParserContext;
> +
> typedef struct JSONMessageParser {
> void (*emit)(void *opaque, QObject *json, Error *err);
> void *opaque;
> diff --git a/qobject/json-parser-int.h b/qobject/json-parser-int.h
> index 8c01f236276..1f435cb8eb2 100644
> --- a/qobject/json-parser-int.h
> +++ b/qobject/json-parser-int.h
> @@ -49,6 +49,9 @@ void json_message_process_token(JSONLexer *lexer, GString
> *input,
>
> /* json-parser.c */
> JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr);
> -QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp);
> +void json_parser_init(JSONParserContext *ctxt, va_list *ap);
> +void json_parser_reset(JSONParserContext *ctxt);
> +QObject *json_parser_feed(JSONParserContext *ctxt, const JSONToken *token,
> Error **errp);
> +void json_parser_destroy(JSONParserContext *ctxt);
>
> #endif
> diff --git a/qobject/json-parser.c b/qobject/json-parser.c
> index f6622b82b0a..3b5edc5bae4 100644
> --- a/qobject/json-parser.c
> +++ b/qobject/json-parser.c
> @@ -31,12 +31,105 @@ struct JSONToken {
> char str[];
> };
>
> -typedef struct JSONParserContext {
> - Error *err;
> - JSONToken *current;
> - GQueue *buf;
> - va_list *ap;
> -} JSONParserContext;
> +/*
> + * The JSON parser is a push parser, returning to the caller after every
> + * token.
The thing that returns after every token is json_parser_feed(), right?
Detail not mentioned here: the value it returns. Leaving that to
json_parser_feed()'s contract feels fine, but pointing from here to
there could be useful.
> Therefore it has an explicit representation of its parser
> + * stack; each stack entry consists of a parser state and a QObject:
> + * - a QList, for an array that is being added to
> + * - a QDict, for a dictionary that is being added to
> + * - a QString, for the key of the next pair that will be added to a QDict
> + *
> + * The stack represents an arbitrary nesting of arrays and dictionaries
> + * (whose next key has been parsed); it can also have a dictionary whose
> + * next key has not been parsed, but that can only happen at the top level.
> + * Because of this, the stack contents are always of the form
> + * "(QList | QDict QString)* QDict?".
> + *
> + * An empty stack represents the beginning of the parsing process, with
> + * start state BEFORE_VALUE.
> + */
> +
> +typedef enum JSONParserState {
> + AFTER_LCURLY,
> + AFTER_LSQUARE,
> + BEFORE_KEY,
> + BEFORE_VALUE,
> + END_OF_KEY,
> + END_OF_VALUE,
> +} JSONParserState;
> +
> +typedef struct JSONParserStackEntry {
> + /*
> + * State when the container is completed or, for the top of the stack,
> + * entry state for the next token.
> + */
> + JSONParserState state;
> +
> + /*
> + * A QString with the last parsed key, or a QList/QDict for the current
> + * container.
> + */
> + QObject *partial;
> +} JSONParserStackEntry;
> +
> +/*
> + * This is the JSON grammar that's parsed, with the state transition and
> + * action at each point of the grammar. While this is not a formal
> + * description, "-> action" represents the pseudocode of the action
> + * and "-> STATE" sets the top stack entry's state to STATE.
> + *
> + * // The initial state is BEFORE_VALUE.
> + * input := value -> END_OF_VALUE -> return parsed value
> + * END_OF_INPUT -> check stack is empty
How can the stack *not* be empty here?
> + *
> + * // entered on BEFORE_VALUE; after any of these rules are processed, the
> + * // parser has completed a QObject and is in the END_OF_VALUE state.
> + * //
> + * // When the parser reaches the END_OF_VALUE state, it examines the
> + * // top of the stack to see if it's coming from "input" (stack empty),
> + * // "array_items" (TOS is a QList) or "dict_pairs" (TOS is a QString; the
> + * // item below will be a QDict). It then proceeds with the corresponding
> + * // actions, which will be one of:
> + * // - return parsed value
> + * // - add value to QList
> + * // - pop QString with the key, add key/value to the QDict
> + * value := literal -> END_OF_VALUE
> + * | '[' -> push empty QList -> AFTER_LSQUARE
> + * after_lsquare -> END_OF_VALUE
> + * | '{' -> push empty QDict -> AFTER_LCURLY
> + * after_lcurly -> END_OF_VALUE
> + *
> + * // non-recursive values, entered on BEFORE_VALUE
> + * literal := INTEGER -> END_OF_VALUE
> + * | FLOAT -> END_OF_VALUE
> + * | KEYWORD -> END_OF_VALUE
> + * | STRING -> END_OF_VALUE
> + * | INTERP -> END_OF_VALUE
> + *
> + * // entered on AFTER_LSQUARE
> + * after_lsquare := ']' -> pop completed QList -> END_OF_VALUE
> + * | ϵ -> BEFORE_VALUE
> + * array_items -> END_OF_VALUE
> + *
> + * // entered on BEFORE_VALUE, with TOS being a QList
> + * array_items := value -> add value to QList -> END_OF_VALUE
> + * (']' -> pop completed QList -> END_OF_VALUE
> + * | ',' -> BEFORE_VALUE
> + * array_items) -> END_OF_VALUE
> + *
> + * // entered on AFTER_LCURLY
> + * after_lcurly := '}' -> pop completed QDict -> END_OF_VALUE
> + * | ϵ -> BEFORE_KEY
> + * dict_pairs -> END_OF_VALUE
> + *
> + * // entered on BEFORE_KEY, with TOS being a QDict
> + * dict_pairs := (STRING | INTERP) -> push QString -> END_OF_KEY
> + * ':' -> BEFORE_VALUE
> + * value -> pop QString + add pair to QDict -> END_OF_VALUE
> + * ('}' -> pop completed QDict -> END_OF_VALUE
> + * | ',' -> BEFORE_KEY
> + * dict_pairs) -> END_OF_VALUE
> + */
This is useful.
It doesn't mention how we do parse errors. Leaving that to
json_parser_feed()'s contract feels fine.
>
> #define BUG_ON(cond) assert(!(cond))
>
> @@ -49,7 +142,26 @@ typedef struct JSONParseCrontext {
> * 4) deal with premature EOI
> */
>
> -static QObject *parse_value(JSONParserContext *ctxt);
> +static inline JSONParserStackEntry *current_entry(JSONParserContext *ctxt)
> +{
> + return g_queue_peek_tail(ctxt->stack);
> +}
> +
> +static void push_entry(JSONParserContext *ctxt, QObject *partial,
> + JSONParserState state)
> +{
> + JSONParserStackEntry *entry = g_new(JSONParserStackEntry, 1);
> + entry->partial = partial;
> + entry->state = state;
> + g_queue_push_tail(ctxt->stack, entry);
> +}
> +
> +static JSONParserStackEntry *pop_entry(JSONParserContext *ctxt)
> +{
> + JSONParserStackEntry *entry = g_queue_pop_tail(ctxt->stack);
> + g_free(entry);
> + return current_entry(ctxt);
> +}
This pops the stack and returns the entry now on top. Slightly
surprising; pop operations commonly return the entry popped from the
stack.
It's this way because you use it like
// invariant: @entry is the entry on top of ctxt->stack, null if empty
value = entry->partial;
entry = pop_entry(ctxt);
Okay. A function comment might reduce surprise.
>
> /**
> * Error handler
> @@ -236,200 +348,10 @@ out:
> return NULL;
> }
>
> -/* Note: the token object returned by parser_context_peek_token or
> - * parser_context_pop_token is deleted as soon as parser_context_pop_token
> - * is called again.
> - */
> -static const JSONToken *parser_context_pop_token(JSONParserContext *ctxt)
> +/* Terminals */
> +
> +static QObject *parse_keyword(JSONParserContext *ctxt, const JSONToken
> *token)
> {
> - g_free(ctxt->current);
> - ctxt->current = g_queue_pop_head(ctxt->buf);
> - return ctxt->current;
> -}
> -
> -static const JSONToken *parser_context_peek_token(JSONParserContext *ctxt)
> -{
> - return g_queue_peek_head(ctxt->buf);
> -}
> -
> -/**
> - * Parsing rules
> - */
> -static int parse_pair(JSONParserContext *ctxt, QDict *dict)
> -{
> - QObject *key_obj = NULL;
> - QString *key;
> - QObject *value;
> - const JSONToken *peek, *token;
> -
> - peek = parser_context_peek_token(ctxt);
> - if (peek == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - key_obj = parse_value(ctxt);
> - key = qobject_to(QString, key_obj);
> - if (!key) {
> - parse_error(ctxt, peek, "key is not a string in object");
> - goto out;
> - }
> -
> - token = parser_context_pop_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - if (token->type != JSON_COLON) {
> - parse_error(ctxt, token, "missing : in object pair");
> - goto out;
> - }
> -
> - value = parse_value(ctxt);
> - if (value == NULL) {
> - parse_error(ctxt, token, "Missing value in dict");
> - goto out;
> - }
> -
> - if (qdict_haskey(dict, qstring_get_str(key))) {
> - parse_error(ctxt, token, "duplicate key");
> - goto out;
> - }
> -
> - qdict_put_obj(dict, qstring_get_str(key), value);
> -
> - qobject_unref(key_obj);
> - return 0;
> -
> -out:
> - qobject_unref(key_obj);
> - return -1;
> -}
> -
> -static QObject *parse_object(JSONParserContext *ctxt)
> -{
> - QDict *dict = NULL;
> - const JSONToken *token, *peek;
> -
> - token = parser_context_pop_token(ctxt);
> - assert(token && token->type == JSON_LCURLY);
> -
> - dict = qdict_new();
> -
> - peek = parser_context_peek_token(ctxt);
> - if (peek == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - if (peek->type != JSON_RCURLY) {
> - if (parse_pair(ctxt, dict) == -1) {
> - goto out;
> - }
> -
> - token = parser_context_pop_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - while (token->type != JSON_RCURLY) {
> - if (token->type != JSON_COMMA) {
> - parse_error(ctxt, token, "expected separator in dict");
> - goto out;
> - }
> -
> - if (parse_pair(ctxt, dict) == -1) {
> - goto out;
> - }
> -
> - token = parser_context_pop_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> - }
> - } else {
> - (void)parser_context_pop_token(ctxt);
> - }
> -
> - return QOBJECT(dict);
> -
> -out:
> - qobject_unref(dict);
> - return NULL;
> -}
> -
> -static QObject *parse_array(JSONParserContext *ctxt)
> -{
> - QList *list = NULL;
> - const JSONToken *token, *peek;
> -
> - token = parser_context_pop_token(ctxt);
> - assert(token && token->type == JSON_LSQUARE);
> -
> - list = qlist_new();
> -
> - peek = parser_context_peek_token(ctxt);
> - if (peek == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - if (peek->type != JSON_RSQUARE) {
> - QObject *obj;
> -
> - obj = parse_value(ctxt);
> - if (obj == NULL) {
> - parse_error(ctxt, token, "expecting value");
> - goto out;
> - }
> -
> - qlist_append_obj(list, obj);
> -
> - token = parser_context_pop_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> -
> - while (token->type != JSON_RSQUARE) {
> - if (token->type != JSON_COMMA) {
> - parse_error(ctxt, token, "expected separator in list");
> - goto out;
> - }
> -
> - obj = parse_value(ctxt);
> - if (obj == NULL) {
> - parse_error(ctxt, token, "expecting value");
> - goto out;
> - }
> -
> - qlist_append_obj(list, obj);
> -
> - token = parser_context_pop_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - goto out;
> - }
> - }
> - } else {
> - (void)parser_context_pop_token(ctxt);
> - }
> -
> - return QOBJECT(list);
> -
> -out:
> - qobject_unref(list);
> - return NULL;
> -}
> -
> -static QObject *parse_keyword(JSONParserContext *ctxt)
> -{
> - const JSONToken *token;
> -
> - token = parser_context_pop_token(ctxt);
> assert(token && token->type == JSON_KEYWORD);
>
> if (!strcmp(token->str, "true")) {
> @@ -443,11 +365,9 @@ static QObject *parse_keyword(JSONParserContext *ctxt)
> return NULL;
> }
>
> -static QObject *parse_interpolation(JSONParserContext *ctxt)
> +static QObject *parse_interpolation(JSONParserContext *ctxt,
> + const JSONToken *token)
> {
> - const JSONToken *token;
> -
> - token = parser_context_pop_token(ctxt);
> assert(token && token->type == JSON_INTERP);
>
> if (!strcmp(token->str, "%p")) {
> @@ -479,11 +399,8 @@ static QObject *parse_interpolation(JSONParserContext
> *ctxt)
> return NULL;
> }
>
> -static QObject *parse_literal(JSONParserContext *ctxt)
> +static QObject *parse_literal(JSONParserContext *ctxt, const JSONToken
> *token)
> {
> - const JSONToken *token;
> -
> - token = parser_context_pop_token(ctxt);
> assert(token);
>
> switch (token->type) {
> @@ -531,35 +448,167 @@ static QObject *parse_literal(JSONParserContext *ctxt)
> }
> }
>
> -static QObject *parse_value(JSONParserContext *ctxt)
> +/* Parsing state machine */
> +
> +static QObject *parse_begin_value(JSONParserContext *ctxt,
> + const JSONToken *token)
> {
> - const JSONToken *token;
> -
> - token = parser_context_peek_token(ctxt);
> - if (token == NULL) {
> - parse_error(ctxt, NULL, "premature EOI");
> - return NULL;
> - }
> -
> switch (token->type) {
> case JSON_LCURLY:
> - return parse_object(ctxt);
> + push_entry(ctxt, QOBJECT(qdict_new()), AFTER_LCURLY);
> + return NULL;
> case JSON_LSQUARE:
> - return parse_array(ctxt);
> + push_entry(ctxt, QOBJECT(qlist_new()), AFTER_LSQUARE);
> + return NULL;
> case JSON_INTERP:
> - return parse_interpolation(ctxt);
> + return parse_interpolation(ctxt, token);
> case JSON_INTEGER:
> case JSON_FLOAT:
> case JSON_STRING:
> - return parse_literal(ctxt);
> + return parse_literal(ctxt, token);
> case JSON_KEYWORD:
> - return parse_keyword(ctxt);
> + return parse_keyword(ctxt, token);
> default:
> parse_error(ctxt, token, "expecting value");
> return NULL;
> }
> }
>
> +static QObject *parse_token(JSONParserContext *ctxt, const JSONToken *token)
> +{
> + JSONParserStackEntry *entry;
> + JSONParserState state;
> + QString *key;
> + QObject *key_obj = NULL, *value = NULL;
> +
> + entry = current_entry(ctxt);
> + state = entry ? entry->state : BEFORE_VALUE;
> + switch (state) {
> + case AFTER_LCURLY:
> + /* Grab '}' for empty object or fall through to BEFORE_KEY */
> + assert(qobject_type(entry->partial) == QTYPE_QDICT);
> + if (token->type == JSON_RCURLY) {
> + value = entry->partial;
> + entry = pop_entry(ctxt);
> + break;
> + }
> + entry->state = BEFORE_KEY;
> + /* fall through */
> +
> + case BEFORE_KEY:
> + /* Expecting object key */
> + assert(qobject_type(entry->partial) == QTYPE_QDICT);
> + if (token->type == JSON_STRING || token->type == JSON_INTERP) {
> + key_obj = parse_begin_value(ctxt, token);
v2 used parse_string() here, which broke interpolation with %s. This
version works.
> + if (!key_obj) {
> + /* parse error happened */
> + return NULL;
> + }
> + }
> + if (!key_obj || qobject_type(key_obj) != QTYPE_QSTRING) {
> + parse_error(ctxt, token, "key is not a string in object");
> + return NULL;
> + }
> +
> + /* Store key in a special entry on the stack */
> + push_entry(ctxt, key_obj, END_OF_KEY);
> + return NULL;
> +
> + case END_OF_KEY:
> + /* Expecting ':' after key */
> + assert(qobject_type(entry->partial) == QTYPE_QSTRING);
> + if (token->type == JSON_COLON) {
> + entry->state = BEFORE_VALUE;
> + } else {
> + parse_error(ctxt, token, "expecting ':'");
> + }
> + return NULL;
> +
> + case AFTER_LSQUARE:
> + /* Grab ']' for empty array or fall through to BEFORE_VALUE */
> + assert(qobject_type(entry->partial) == QTYPE_QLIST);
> + if (token->type == JSON_RSQUARE) {
> + value = entry->partial;
> + entry = pop_entry(ctxt);
> + break;
> + }
> + entry->state = BEFORE_VALUE;
> + /* fall through */
> +
> + case BEFORE_VALUE:
> + /* Expecting value */
> + assert(!entry || qobject_type(entry->partial) != QTYPE_QDICT);
> + value = parse_begin_value(ctxt, token);
> + if (!value) {
> + /* Error or '['/'{' */
> + return NULL;
> + }
> + /* Return value or insert it into a container */
> + break;
> +
> + case END_OF_VALUE:
> + /* Grab ',' or ']' for array; ',' or '}' for object */
> + if (qobject_to(QList, entry->partial)) {
> + /* Array */
> + if (token->type != JSON_RSQUARE) {
> + if (token->type == JSON_COMMA) {
> + entry->state = BEFORE_VALUE;
> + } else {
> + parse_error(ctxt, token, "expected ',' or ']'");
> + }
> + return NULL;
> + }
> + } else if (qobject_to(QDict, entry->partial)) {
> + /* Object */
> + if (token->type != JSON_RCURLY) {
> + if (token->type == JSON_COMMA) {
> + entry->state = BEFORE_KEY;
> + } else {
> + parse_error(ctxt, token, "expected ',' or '}'");
> + }
> + return NULL;
> + }
> + } else {
> + g_assert_not_reached();
> + }
> +
> + /* Got ']' or '}'; return full value or insert into parent container
> */
> + value = entry->partial;
> + entry = pop_entry(ctxt);
> + break;
> + }
> +
> + assert(value);
> + if (entry == NULL) {
> + /* The toplevel value is complete. */
Maybe
/* Parse stack is empty, top level value is complete */
> + return value;
> + }
> +
Suggest
/*
* Parse stack is not empty.
* If we're parsing an object, it's QString (key) on top of
* QDict. Pop off key, and store (key, value) in QDict.
* If we're parsing an array, it's QList. Store value in it.
*/
> + key = qobject_to(QString, entry->partial);
> + if (key) {
> + const char *key_str;
> + QDict *dict;
> +
> + entry = pop_entry(ctxt);
> + dict = qobject_to(QDict, entry->partial);
> + assert(dict);
> + key_str = qstring_get_str(key);
> + if (qdict_haskey(dict, key_str)) {
> + parse_error(ctxt, token, "duplicate key");
> + qobject_unref(value);
> + return NULL;
> + }
> + qdict_put_obj(dict, key_str, value);
> + qobject_unref(key);
> + } else {
> + /* Add to array */
> + qlist_append_obj(qobject_to(QList, entry->partial), value);
> + }
> +
> + entry->state = END_OF_VALUE;
> + return NULL;
> +}
> +
> JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr)
> {
> JSONToken *token = g_malloc(sizeof(JSONToken) + tokstr->len + 1);
> @@ -572,20 +621,56 @@ JSONToken *json_token(JSONTokenType type, int x, int y,
> GString *tokstr)
> return token;
> }
>
> -QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp)
> +void json_parser_reset(JSONParserContext *ctxt)
> {
> - JSONParserContext ctxt = { .buf = tokens, .ap = ap };
> - QObject *result;
> + JSONParserStackEntry *entry;
>
> - result = parse_value(&ctxt);
> - assert(ctxt.err || g_queue_is_empty(ctxt.buf));
> -
> - error_propagate(errp, ctxt.err);
> -
> - while (!g_queue_is_empty(ctxt.buf)) {
> - parser_context_pop_token(&ctxt);
> + ctxt->err = NULL;
> + while ((entry = g_queue_pop_tail(ctxt->stack)) != NULL) {
> + qobject_unref(entry->partial);
> + g_free(entry);
> }
> - g_free(ctxt.current);
> +}
>
> +void json_parser_init(JSONParserContext *ctxt, va_list *ap)
> +{
> + ctxt->stack = g_queue_new();
> + ctxt->ap = ap;
> + json_parser_reset(ctxt);
> +}
> +
> +void json_parser_destroy(JSONParserContext *ctxt)
> +{
> + json_parser_reset(ctxt);
> + g_queue_free(ctxt->stack);
> + ctxt->stack = NULL;
> +}
> +
> +/*
> + * Advance the parser based on the token that is passed.
> + * Return the finished toplevel value if the token completes it.
My dictionary wants "top level" or "top-level".
> + * If an error is returned, the function must not be called without
> + * first resetting the parser.
> + */
> +QObject *json_parser_feed(JSONParserContext *ctxt, const JSONToken *token,
> + Error **errp)
> +{
> + QObject *result = NULL;
> +
> + assert(!ctxt->err);
> + switch (token->type) {
> + case JSON_END_OF_INPUT:
> + /* Check for premature end of input */
> + if (!g_queue_is_empty(ctxt->stack)) {
> + parse_error(ctxt, token, "premature end of input");
> + }
> + break;
> +
> + default:
> + result = parse_token(ctxt, token);
> + break;
> + }
> +
> + error_propagate(errp, ctxt->err);
> return result;
> }
> diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
> index b93d97b995f..6c93e6fd78d 100644
> --- a/qobject/json-streamer.c
> +++ b/qobject/json-streamer.c
> @@ -32,6 +32,7 @@ void json_message_process_token(JSONLexer *lexer, GString
> *input,
> JSONTokenType type, int x, int y)
> {
> JSONMessageParser *parser = container_of(lexer, JSONMessageParser,
> lexer);
> + JSONParserContext ctxt;
> QObject *json = NULL;
> Error *err = NULL;
> JSONToken *token;
> @@ -56,8 +57,7 @@ void json_message_process_token(JSONLexer *lexer, GString
> *input,
> if (g_queue_is_empty(&parser->tokens)) {
> return;
> }
> - json = json_parser_parse(&parser->tokens, parser->ap, &err);
> - goto out_emit;
> + break;
> default:
> break;
> }
> @@ -85,11 +85,24 @@ void json_message_process_token(JSONLexer *lexer, GString
> *input,
> g_queue_push_tail(&parser->tokens, token);
>
> if ((parser->brace_count > 0 || parser->bracket_count > 0)
> - && parser->brace_count >= 0 && parser->bracket_count >= 0) {
> + && parser->brace_count >= 0 && parser->bracket_count >= 0
> + && type != JSON_END_OF_INPUT) {
> return;
> }
>
> - json = json_parser_parse(&parser->tokens, parser->ap, &err);
> + json_parser_init(&ctxt, parser->ap);
> +
> + /* Process all tokens in the queue */
> + while (!g_queue_is_empty(&parser->tokens)) {
> + token = g_queue_pop_head(&parser->tokens);
> + json = json_parser_feed(&ctxt, token, &err);
> + g_free(token);
> + if (json || err) {
> + break;
> + }
> + }
> +
> + json_parser_destroy(&ctxt);
>
> out_emit:
> parser->brace_count = 0;
