Hi, this is a spin-off patch from Fabien COELHO's backslash-continuations. The major concept of this patch is making usage of psql's scanner to get rid of home-grown scanner of pgbench to make multi-statement feature available for pgbench custom scripts.
This patch does the following things. - Modify psqlscan.l so that unnecessary functions of it can be masked when used in other modules like pgbench. - Modify pgbench to use psqlscan.l so that the following features available in pgbench. - multi-statement in custom scripts. - natural continuation of SQL statements in costom scripts. - backslash-continuation for pgbench metacommands in costom scripts. The patch consists of following files. - 0001-Prepare-to-share-psqlscan-with-pgbench.patch Modifies psqlscan.l in psql as the preparation. - 0002-Make-use-of-psqlscan-for-parsing-of-custom-script.patch Modifies pgbench to use psqlscan.l. - 0003-Change-MSVC-Build-script.patch Modify MSVC build script. - 0004-Change-the-way-to-hold-command-list.patch Get rid of double-format of internal command list. This changes the way of holding command list to linked list totally. regards,
>From c8830544312308b42d9ce7fc5793519c32237ba5 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyot...@lab.ntt.co.jp> Date: Thu, 23 Jul 2015 20:44:37 +0900 Subject: [PATCH 1/4] Prepare to share psqlscan with pgbench. Eliminate direct usage of pset variables and enable parts unnecessary for other than psql to be disabled by defining OUTSIDE_PSQL. --- src/bin/psql/mainloop.c | 6 ++-- src/bin/psql/psqlscan.h | 14 +++++---- src/bin/psql/psqlscan.l | 79 ++++++++++++++++++++++++++++++++----------------- src/bin/psql/startup.c | 4 +-- 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/src/bin/psql/mainloop.c b/src/bin/psql/mainloop.c index b6cef94..e98cb94 100644 --- a/src/bin/psql/mainloop.c +++ b/src/bin/psql/mainloop.c @@ -233,7 +233,8 @@ MainLoop(FILE *source) /* * Parse line, looking for command separators. */ - psql_scan_setup(scan_state, line, strlen(line)); + psql_scan_setup(scan_state, line, strlen(line), + pset.db, pset.vars, pset.encoding); success = true; line_saved_in_history = false; @@ -373,7 +374,8 @@ MainLoop(FILE *source) resetPQExpBuffer(query_buf); /* reset parsing state since we are rescanning whole line */ psql_scan_reset(scan_state); - psql_scan_setup(scan_state, line, strlen(line)); + psql_scan_setup(scan_state, line, strlen(line), + pset.db, pset.vars, pset.encoding); line_saved_in_history = false; prompt_status = PROMPT_READY; } diff --git a/src/bin/psql/psqlscan.h b/src/bin/psql/psqlscan.h index 55070ca..4bf8dcb 100644 --- a/src/bin/psql/psqlscan.h +++ b/src/bin/psql/psqlscan.h @@ -11,7 +11,11 @@ #include "pqexpbuffer.h" #include "prompt.h" - +#if !defined OUTSIDE_PSQL +#include "variables.h" +#else +typedef int * VariableSpace; +#endif /* Abstract type for lexer's internal state */ typedef struct PsqlScanStateData *PsqlScanState; @@ -36,12 +40,11 @@ enum slash_option_type OT_NO_EVAL /* no expansion of backticks or variables */ }; - extern PsqlScanState psql_scan_create(void); extern void psql_scan_destroy(PsqlScanState state); -extern void psql_scan_setup(PsqlScanState state, - const char *line, int line_len); +extern void psql_scan_setup(PsqlScanState state, const char *line, int line_len, + PGconn *db, VariableSpace vars, int encoding); extern void psql_scan_finish(PsqlScanState state); extern PsqlScanResult psql_scan(PsqlScanState state, @@ -52,6 +55,7 @@ extern void psql_scan_reset(PsqlScanState state); extern bool psql_scan_in_quote(PsqlScanState state); +#if !defined OUTSIDE_PSQL extern char *psql_scan_slash_command(PsqlScanState state); extern char *psql_scan_slash_option(PsqlScanState state, @@ -60,5 +64,5 @@ extern char *psql_scan_slash_option(PsqlScanState state, bool semicolon); extern void psql_scan_slash_command_end(PsqlScanState state); - +#endif /* if !defined OUTSIDE_PSQL */ #endif /* PSQLSCAN_H */ diff --git a/src/bin/psql/psqlscan.l b/src/bin/psql/psqlscan.l index be059ab..f9a19cd 100644 --- a/src/bin/psql/psqlscan.l +++ b/src/bin/psql/psqlscan.l @@ -43,11 +43,6 @@ #include <ctype.h> -#include "common.h" -#include "settings.h" -#include "variables.h" - - /* * We use a stack of flex buffers to handle substitution of psql variables. * Each stacked buffer contains the as-yet-unread text from one psql variable. @@ -81,10 +76,12 @@ typedef struct PsqlScanStateData const char *scanline; /* current input line at outer level */ /* safe_encoding, curline, refline are used by emit() to replace FFs */ + PGconn *db; /* active connection */ int encoding; /* encoding being used now */ bool safe_encoding; /* is current encoding "safe"? */ const char *curline; /* actual flex input string for cur buf */ const char *refline; /* original data for cur buffer */ + VariableSpace vars; /* "shell variable" repository */ /* * All this state lives across successive input lines, until explicitly @@ -126,6 +123,15 @@ static void escape_variable(bool as_ident); #define ECHO emit(yytext, yyleng) +/* Provide dummy macros when no use of psql variables */ +#if defined OUTSIDE_PSQL +#define GetVariable(space,name) NULL +#define standard_strings() true +#define psql_error(fmt,...) do { \ + fprintf(stderr, "psql_error is called. abort.\n");\ + exit(1);\ +} while(0) +#endif %} %option 8bit @@ -736,11 +742,14 @@ other . :{variable_char}+ { /* Possible psql variable substitution */ - char *varname; - const char *value; + char *varname = NULL; + const char *value = NULL; - varname = extract_substring(yytext + 1, yyleng - 1); - value = GetVariable(pset.vars, varname); + if (cur_state->vars) + { + varname = extract_substring(yytext + 1, yyleng - 1); + value = GetVariable(cur_state->vars, varname); + } if (value) { @@ -769,7 +778,8 @@ other . ECHO; } - free(varname); + if (varname) + free(varname); } :'{variable_char}+' { @@ -1033,9 +1043,12 @@ other . char *varname; const char *value; - varname = extract_substring(yytext + 1, yyleng - 1); - value = GetVariable(pset.vars, varname); - free(varname); + if (cur_state->vars) + { + varname = extract_substring(yytext + 1, yyleng - 1); + value = GetVariable(cur_state->vars, varname); + free(varname); + } /* * The variable value is just emitted without any @@ -1227,17 +1240,19 @@ psql_scan_destroy(PsqlScanState state) * or freed until after psql_scan_finish is called. */ void -psql_scan_setup(PsqlScanState state, - const char *line, int line_len) +psql_scan_setup(PsqlScanState state, const char *line, int line_len, + PGconn *db, VariableSpace vars, int encoding) { /* Mustn't be scanning already */ Assert(state->scanbufhandle == NULL); Assert(state->buffer_stack == NULL); /* Do we need to hack the character set encoding? */ - state->encoding = pset.encoding; + state->encoding = encoding; state->safe_encoding = pg_valid_server_encoding_id(state->encoding); + state->vars = vars; + /* needed for prepare_buffer */ cur_state = state; @@ -1459,6 +1474,7 @@ psql_scan_in_quote(PsqlScanState state) return state->start_state != INITIAL; } +#if !defined OUTSIDE_PSQL /* * Scan the command name of a psql backslash command. This should be called * after psql_scan() returns PSCAN_BACKSLASH. It is assumed that the input @@ -1615,7 +1631,7 @@ psql_scan_slash_option(PsqlScanState state, { if (!inquotes && type == OT_SQLID) *cp = pg_tolower((unsigned char) *cp); - cp += PQmblen(cp, pset.encoding); + cp += PQmblen(cp, cur_state->encoding); } } } @@ -1744,6 +1760,14 @@ evaluate_backtick(void) termPQExpBuffer(&cmd_output); } +#else +static void +evaluate_backtick(void) +{ + fprintf(stderr, "Unexpected call of evaluate_backtick.\n"); + exit(1); +} +#endif /* if !defined OUTSIDE_PSQL*/ /* * Push the given string onto the stack of stuff to scan. @@ -1944,15 +1968,18 @@ escape_variable(bool as_ident) char *varname; const char *value; - /* Variable lookup. */ - varname = extract_substring(yytext + 2, yyleng - 3); - value = GetVariable(pset.vars, varname); - free(varname); + /* Variable lookup if possible. */ + if (cur_state->vars && cur_state->db) + { + varname = extract_substring(yytext + 2, yyleng - 3); + value = GetVariable(cur_state->vars, varname); + free(varname); + } /* Escaping. */ if (value) { - if (!pset.db) + if (!cur_state->db) psql_error("can't escape without active connection\n"); else { @@ -1960,16 +1987,14 @@ escape_variable(bool as_ident) if (as_ident) escaped_value = - PQescapeIdentifier(pset.db, value, strlen(value)); + PQescapeIdentifier(cur_state->db, value, strlen(value)); else escaped_value = - PQescapeLiteral(pset.db, value, strlen(value)); + PQescapeLiteral(cur_state->db, value, strlen(value)); if (escaped_value == NULL) { - const char *error = PQerrorMessage(pset.db); - - psql_error("%s", error); + psql_error("%s", PQerrorMessage(cur_state->db)); } else { diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 28ba75a..c143dfe 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -305,8 +305,8 @@ main(int argc, char *argv[]) scan_state = psql_scan_create(); psql_scan_setup(scan_state, - options.action_string, - strlen(options.action_string)); + options.action_string, strlen(options.action_string), + pset.db, pset.vars, pset.encoding); successResult = HandleSlashCmds(scan_state, NULL) != PSQL_CMD_ERROR ? EXIT_SUCCESS : EXIT_FAILURE; -- 1.8.3.1
>From a4f0459773c7d827a691f8b3c4b776346cd45d1f Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyot...@lab.ntt.co.jp> Date: Fri, 24 Jul 2015 10:58:23 +0900 Subject: [PATCH 2/4] Make use of psqlscan for parsing of custom script. Make use of psqlscan instead of the home-made parser allowing backslash continuation for backslash commands, multiline SQL statements and SQL multi statement in custom scripts. --- src/bin/pgbench/Makefile | 16 +- src/bin/pgbench/pgbench.c | 478 +++++++++++++++++++++++++++++++--------------- 2 files changed, 341 insertions(+), 153 deletions(-) diff --git a/src/bin/pgbench/Makefile b/src/bin/pgbench/Makefile index 18fdf58..a0a736b 100644 --- a/src/bin/pgbench/Makefile +++ b/src/bin/pgbench/Makefile @@ -5,11 +5,13 @@ PGAPPICON = win32 subdir = src/bin/pgbench top_builddir = ../../.. +psqlincdir = ../psql include $(top_builddir)/src/Makefile.global OBJS = pgbench.o exprparse.o $(WIN32RES) -override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) + +override CPPFLAGS := -DOUTSIDE_PSQL -I. -I$(srcdir) -I$(libpq_srcdir) -I$(psqlincdir) $(CPPFLAGS) ifneq ($(PORTNAME), win32) override CFLAGS += $(PTHREAD_CFLAGS) @@ -18,6 +20,16 @@ endif all: pgbench +psqlscan.c: FLEXFLAGS = -Cfe -p -p +psqlscan.c: FLEX_NO_BACKUP=yes + +psqlscan.l: % : $(top_srcdir)/src/bin/psql/% + rm -f $@ && $(LN_S) $< . + +psqlscan.c: psqlscan.l + +pgbench.o: psqlscan.c + pgbench: $(OBJS) | submake-libpq submake-libpgport $(CC) $(CFLAGS) $^ $(libpq_pgport) $(PTHREAD_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) @@ -39,4 +51,4 @@ clean distclean: rm -f pgbench$(X) $(OBJS) maintainer-clean: distclean - rm -f exprparse.c exprscan.c + rm -f exprparse.c exprscan.c psqlscan.l psqlscan.c diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 30e8d2a..b6fd399 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -54,6 +54,7 @@ #endif #include "pgbench.h" +#include "psqlscan.h" #define ERRCODE_UNDEFINED_TABLE "42P01" @@ -264,7 +265,7 @@ typedef enum QueryMode static QueryMode querymode = QUERY_SIMPLE; static const char *QUERYMODE[] = {"simple", "extended", "prepared"}; -typedef struct +typedef struct Command_t { char *line; /* full text of command line */ int command_num; /* unique index of this Command struct */ @@ -273,6 +274,7 @@ typedef struct char *argv[MAX_ARGS]; /* command word list */ int cols[MAX_ARGS]; /* corresponding column starting from 1 */ PgBenchExpr *expr; /* parsed expression */ + struct Command_t *next; /* more command if any, for multistatements */ } Command; typedef struct @@ -295,6 +297,21 @@ typedef struct double sum2_lag; /* sum(lag*lag) */ } AggVals; +typedef enum +{ + PS_IDLE, + PS_IN_STATEMENT, + PS_IN_BACKSLASH_CMD +} ParseState; + +typedef struct ParseInfo +{ + PsqlScanState scan_state; + PQExpBuffer outbuf; + ParseState mode; +} ParseInfoData; +typedef ParseInfoData *ParseInfo; + static Command **sql_files[MAX_FILES]; /* SQL script files */ static int num_files; /* number of script files */ static int num_commands = 0; /* total number of Command structs */ @@ -2224,217 +2241,348 @@ syntax_error(const char *source, const int lineno, exit(1); } -/* Parse a command; return a Command struct, or NULL if it's a comment */ +static ParseInfo +createParseInfo(void) +{ + ParseInfo ret = (ParseInfo) pg_malloc(sizeof(ParseInfoData)); + + ret->scan_state = psql_scan_create(); + ret->outbuf = createPQExpBuffer(); + ret->mode = PS_IDLE; + + return ret; +} + +#define parse_reset_outbuf(pcs) resetPQExpBuffer((pcs)->outbuf) +#define parse_finish_scan(pcs) psql_scan_finish((pcs)->scan_state) + +/* copy a string after removing newlines and collapsing whitespaces */ +static char * +strdup_nonl(const char *in) +{ + char *ret, *p, *q; + + ret = pg_strdup(in); + + /* Replace newlines into spaces */ + for (p = ret ; *p ; p++) + if (*p == '\n') *p = ' '; + + /* collapse successive spaces */ + for (p = q = ret ; *p ; p++, q++) + { + while (isspace(*p) && isspace(*(p + 1))) p++; + if (p > q) *q = *p; + } + *q = '\0'; + + return ret; +} + +/* Parse a backslash command; return a Command struct */ static Command * -process_commands(char *buf, const char *source, const int lineno) +process_backslash_commands(ParseInfo proc_state, char *buf, + const char *source, const int lineno) { const char delim[] = " \f\n\r\t\v"; Command *my_commands; int j; char *p, + *start, *tok; - - /* Make the string buf end at the next newline */ - if ((p = strchr(buf, '\n')) != NULL) - *p = '\0'; + int max_args = -1; /* Skip leading whitespace */ p = buf; while (isspace((unsigned char) *p)) p++; + start = p; + + if (proc_state->mode != PS_IN_BACKSLASH_CMD) + { + if (*p != '\\') + return NULL; + + /* This is the first line of a backslash command */ + proc_state->mode = PS_IN_BACKSLASH_CMD; + } + + /* + * Make the string buf end at the next newline, or move to just after the + * end of line + */ + if ((p = strchr(start, '\n')) != NULL) + *p = '\0'; + else + p = start + strlen(start); + + /* continued line ends with a backslash */ + if (*(--p) == '\\') + { + *p-- = '\0'; + appendPQExpBufferStr(proc_state->outbuf, start); + + /* Add a delimiter at the end of the line if necessary */ + if (!isspace(*p)) + appendPQExpBufferChar(proc_state->outbuf, ' '); - /* If the line is empty or actually a comment, we're done */ - if (*p == '\0' || strncmp(p, "--", 2) == 0) return NULL; + } + + appendPQExpBufferStr(proc_state->outbuf, start); + proc_state->mode = PS_IDLE; + + /* Start parsing the backslash command */ + + p = proc_state->outbuf->data; /* Allocate and initialize Command structure */ my_commands = (Command *) pg_malloc(sizeof(Command)); - my_commands->line = pg_strdup(buf); + my_commands->line = pg_strdup(p); my_commands->command_num = num_commands++; - my_commands->type = 0; /* until set */ + my_commands->type = META_COMMAND; my_commands->argc = 0; + my_commands->next = NULL; - if (*p == '\\') - { - int max_args = -1; + j = 0; + tok = strtok(++p, delim); - my_commands->type = META_COMMAND; + if (tok != NULL && pg_strcasecmp(tok, "set") == 0) + max_args = 2; - j = 0; - tok = strtok(++p, delim); + while (tok != NULL) + { + my_commands->cols[j] = tok - buf + 1; + my_commands->argv[j++] = pg_strdup(tok); + my_commands->argc++; + if (max_args >= 0 && my_commands->argc >= max_args) + tok = strtok(NULL, ""); + else + tok = strtok(NULL, delim); + } + parse_reset_outbuf(proc_state); - if (tok != NULL && pg_strcasecmp(tok, "set") == 0) - max_args = 2; + if (pg_strcasecmp(my_commands->argv[0], "setrandom") == 0) + { + /* + * parsing: \setrandom variable min max [uniform] \setrandom + * variable min max (gaussian|exponential) threshold + */ - while (tok != NULL) + if (my_commands->argc < 4) { - my_commands->cols[j] = tok - buf + 1; - my_commands->argv[j++] = pg_strdup(tok); - my_commands->argc++; - if (max_args >= 0 && my_commands->argc >= max_args) - tok = strtok(NULL, ""); - else - tok = strtok(NULL, delim); + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "missing arguments", NULL, -1); } - if (pg_strcasecmp(my_commands->argv[0], "setrandom") == 0) - { - /* - * parsing: \setrandom variable min max [uniform] \setrandom - * variable min max (gaussian|exponential) threshold - */ + /* argc >= 4 */ - if (my_commands->argc < 4) + if (my_commands->argc == 4 || /* uniform without/with + * "uniform" keyword */ + (my_commands->argc == 5 && + pg_strcasecmp(my_commands->argv[4], "uniform") == 0)) + { + /* nothing to do */ + } + else if ( /* argc >= 5 */ + (pg_strcasecmp(my_commands->argv[4], "gaussian") == 0) || + (pg_strcasecmp(my_commands->argv[4], "exponential") == 0)) + { + if (my_commands->argc < 6) { syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing arguments", NULL, -1); - } - - /* argc >= 4 */ - - if (my_commands->argc == 4 || /* uniform without/with - * "uniform" keyword */ - (my_commands->argc == 5 && - pg_strcasecmp(my_commands->argv[4], "uniform") == 0)) - { - /* nothing to do */ + "missing threshold argument", my_commands->argv[4], -1); } - else if ( /* argc >= 5 */ - (pg_strcasecmp(my_commands->argv[4], "gaussian") == 0) || - (pg_strcasecmp(my_commands->argv[4], "exponential") == 0)) - { - if (my_commands->argc < 6) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing threshold argument", my_commands->argv[4], -1); - } - else if (my_commands->argc > 6) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "too many arguments", my_commands->argv[4], - my_commands->cols[6]); - } - } - else /* cannot parse, unexpected arguments */ + else if (my_commands->argc > 6) { syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "unexpected argument", my_commands->argv[4], - my_commands->cols[4]); + "too many arguments", my_commands->argv[4], + my_commands->cols[6]); } } - else if (pg_strcasecmp(my_commands->argv[0], "set") == 0) + else /* cannot parse, unexpected arguments */ { - if (my_commands->argc < 3) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing argument", NULL, -1); - } + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "unexpected argument", my_commands->argv[4], + my_commands->cols[4]); + } + } + else if (pg_strcasecmp(my_commands->argv[0], "set") == 0) + { + if (my_commands->argc < 3) + { + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "missing argument", NULL, -1); + } - expr_scanner_init(my_commands->argv[2], source, lineno, - my_commands->line, my_commands->argv[0], - my_commands->cols[2] - 1); + expr_scanner_init(my_commands->argv[2], source, lineno, + my_commands->line, my_commands->argv[0], + my_commands->cols[2] - 1); - if (expr_yyparse() != 0) - { - /* dead code: exit done from syntax_error called by yyerror */ - exit(1); - } + if (expr_yyparse() != 0) + { + /* dead code: exit done from syntax_error called by yyerror */ + exit(1); + } - my_commands->expr = expr_parse_result; + my_commands->expr = expr_parse_result; - expr_scanner_finish(); - } - else if (pg_strcasecmp(my_commands->argv[0], "sleep") == 0) + expr_scanner_finish(); + } + else if (pg_strcasecmp(my_commands->argv[0], "sleep") == 0) + { + if (my_commands->argc < 2) { - if (my_commands->argc < 2) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing argument", NULL, -1); - } - - /* - * Split argument into number and unit to allow "sleep 1ms" etc. - * We don't have to terminate the number argument with null - * because it will be parsed with atoi, which ignores trailing - * non-digit characters. - */ - if (my_commands->argv[1][0] != ':') - { - char *c = my_commands->argv[1]; + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "missing argument", NULL, -1); + } - while (isdigit((unsigned char) *c)) - c++; - if (*c) - { - my_commands->argv[2] = c; - if (my_commands->argc < 3) - my_commands->argc = 3; - } - } + /* + * Split argument into number and unit to allow "sleep 1ms" etc. We + * don't have to terminate the number argument with null because it + * will be parsed with atoi, which ignores trailing non-digit + * characters. + */ + if (my_commands->argv[1][0] != ':') + { + char *c = my_commands->argv[1]; - if (my_commands->argc >= 3) + while (isdigit((unsigned char) *c)) + c++; + if (*c) { - if (pg_strcasecmp(my_commands->argv[2], "us") != 0 && - pg_strcasecmp(my_commands->argv[2], "ms") != 0 && - pg_strcasecmp(my_commands->argv[2], "s") != 0) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "unknown time unit, must be us, ms or s", - my_commands->argv[2], my_commands->cols[2]); - } + my_commands->argv[2] = c; + if (my_commands->argc < 3) + my_commands->argc = 3; } - - /* this should be an error?! */ - for (j = 3; j < my_commands->argc; j++) - fprintf(stderr, "%s: extra argument \"%s\" ignored\n", - my_commands->argv[0], my_commands->argv[j]); } - else if (pg_strcasecmp(my_commands->argv[0], "setshell") == 0) + + if (my_commands->argc >= 3) { - if (my_commands->argc < 3) + if (pg_strcasecmp(my_commands->argv[2], "us") != 0 && + pg_strcasecmp(my_commands->argv[2], "ms") != 0 && + pg_strcasecmp(my_commands->argv[2], "s") != 0) { syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing argument", NULL, -1); + "unknown time unit, must be us, ms or s", + my_commands->argv[2], my_commands->cols[2]); } } - else if (pg_strcasecmp(my_commands->argv[0], "shell") == 0) + + /* this should be an error?! */ + for (j = 3; j < my_commands->argc; j++) + fprintf(stderr, "%s: extra argument \"%s\" ignored\n", + my_commands->argv[0], my_commands->argv[j]); + } + else if (pg_strcasecmp(my_commands->argv[0], "setshell") == 0) + { + if (my_commands->argc < 3) { - if (my_commands->argc < 1) - { - syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "missing command", NULL, -1); - } + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "missing argument", NULL, -1); } - else + } + else if (pg_strcasecmp(my_commands->argv[0], "shell") == 0) + { + if (my_commands->argc < 1) { syntax_error(source, lineno, my_commands->line, my_commands->argv[0], - "invalid command", NULL, -1); + "missing command", NULL, -1); } } else { - my_commands->type = SQL_COMMAND; + syntax_error(source, lineno, my_commands->line, my_commands->argv[0], + "invalid command", NULL, -1); + } + + return my_commands; +} + +/* Parse a input line, return non-null if any command terminates. */ +static Command * +process_commands(ParseInfo proc_state, char *buf, + const char *source, const int lineno) +{ + Command *command = NULL; + Command *retcomd = NULL; + PsqlScanState scan_state = proc_state->scan_state; + promptStatus_t prompt_status = PROMPT_READY; /* dummy */ + PQExpBuffer qbuf = proc_state->outbuf; + PsqlScanResult scan_result; + + if (proc_state->mode != PS_IN_STATEMENT) + { + command = process_backslash_commands(proc_state, buf, source, lineno); + + /* go to next line for continuation line of backslash command. */ + if (command != NULL || proc_state->mode == PS_IN_BACKSLASH_CMD) + return command; + } + + /* Parse statements */ + psql_scan_setup(scan_state, buf, strlen(buf), NULL, NULL, 0); + +next_command: + scan_result = psql_scan(scan_state, qbuf, &prompt_status); + + if (scan_result == PSCAN_SEMICOLON) + { + proc_state->mode = PS_IDLE; + /* + * Command is terminated. Fill the struct. + */ + command = (Command*) pg_malloc(sizeof(Command)); + command->line = strdup_nonl(qbuf->data); + command->command_num = num_commands++; + command->type = SQL_COMMAND; + command->argc = 0; + command->next = NULL; + + /* Put this command at the end of returning command chain */ + if (!retcomd) + retcomd = command; + else + { + Command *pcomm = retcomd; + while (pcomm->next) pcomm = pcomm->next; + pcomm->next = command; + } switch (querymode) { - case QUERY_SIMPLE: - my_commands->argv[0] = pg_strdup(p); - my_commands->argc++; - break; - case QUERY_EXTENDED: - case QUERY_PREPARED: - if (!parseQuery(my_commands, p)) - exit(1); - break; - default: + case QUERY_SIMPLE: + command->argv[0] = pg_strdup(qbuf->data); + command->argc++; + break; + case QUERY_EXTENDED: + case QUERY_PREPARED: + if (!parseQuery(command, qbuf->data)) exit(1); + break; + default: + exit(1); } + + parse_reset_outbuf(proc_state); + + /* Ask for the next statement in this line */ + goto next_command; + } + else if (scan_result == PSCAN_BACKSLASH) + { + fprintf(stderr, "Unexpected backslash in SQL statement: %s:%d\n", source, lineno); + exit(1); } - return my_commands; + proc_state->mode = PS_IN_STATEMENT; + psql_scan_finish(scan_state); + + return retcomd; } + /* * Read a line from fd, and return it in a malloc'd buffer. * Return NULL at EOF. @@ -2489,6 +2637,7 @@ process_file(char *filename) index; char *buf; int alloc_num; + ParseInfo proc_state = createParseInfo(); if (num_files >= MAX_FILES) { @@ -2509,33 +2658,47 @@ process_file(char *filename) return false; } + proc_state->mode = PS_IDLE; + lineno = 0; index = 0; while ((buf = read_line_from_file(fd)) != NULL) { - Command *command; + Command *command = NULL; lineno += 1; - command = process_commands(buf, filename, lineno); - + command = process_commands(proc_state, buf, filename, lineno); free(buf); if (command == NULL) + { + /* + * command is NULL when psql_scan returns PSCAN_EOL or + * PSCAN_INCOMPLETE. Immediately ask for the next line for the + * cases. + */ continue; + } - my_commands[index] = command; - index++; + while (command) + { + my_commands[index++] = command; + command = command->next; + } - if (index >= alloc_num) + if (index > alloc_num) { alloc_num += COMMANDS_ALLOC_NUM; - my_commands = pg_realloc(my_commands, sizeof(Command *) * alloc_num); + my_commands = pg_realloc(my_commands, + sizeof(Command *) * alloc_num); } } fclose(fd); + parse_finish_scan(proc_state); + my_commands[index] = NULL; sql_files[num_files++] = my_commands; @@ -2553,6 +2716,7 @@ process_builtin(char *tb, const char *source) index; char buf[BUFSIZ]; int alloc_num; + ParseInfo proc_state = createParseInfo(); alloc_num = COMMANDS_ALLOC_NUM; my_commands = (Command **) pg_malloc(sizeof(Command *) * alloc_num); @@ -2579,10 +2743,12 @@ process_builtin(char *tb, const char *source) lineno += 1; - command = process_commands(buf, source, lineno); + command = process_commands(proc_state, buf, source, lineno); if (command == NULL) continue; + /* builtin doesn't need multistatements */ + Assert(command->next == NULL); my_commands[index] = command; index++; @@ -2594,6 +2760,7 @@ process_builtin(char *tb, const char *source) } my_commands[index] = NULL; + parse_finish_scan(proc_state); return my_commands; } @@ -3934,3 +4101,12 @@ pthread_join(pthread_t th, void **thread_return) } #endif /* WIN32 */ + +/* + * psqlscan.c is #include'd here instead of being compiled on its own. + * This is because we need postgres_fe.h to be read before any system + * include files, else things tend to break on platforms that have + * multiple infrastructures for stdio.h and so on. flex is absolutely + * uncooperative about that, so we can't compile psqlscan.c on its own. + */ +#include "psqlscan.c" -- 1.8.3.1
>From 82c439bc2ae53fc8190ed372f77a67428fb15c60 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyot...@lab.ntt.co.jp> Date: Tue, 4 Aug 2015 20:54:28 +0900 Subject: [PATCH 3/4] Change MSVC Build script --- src/tools/msvc/Mkvcbuild.pm | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 3abbb4c..f018a29 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -68,7 +68,7 @@ my $frontend_extrasource = { [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ], }; my @frontend_excludes = ( 'pgevent', 'pg_basebackup', 'pg_rewind', 'pg_dump', - 'pg_xlogdump', 'scripts'); + 'pg_xlogdump', 'pgbench', 'scripts'); sub mkvcbuild { @@ -671,6 +671,14 @@ sub mkvcbuild } $pg_xlogdump->AddFile('src/backend/access/transam/xlogreader.c'); + # fix up pg_xlogdump once it's been set up + # files symlinked on Unix are copied on windows + my $pgbench = AddSimpleFrontend('pgbench'); + $pgbench->AddDefine('FRONTEND'); + $pgbench->AddDefine('OUTSIDE_PSQL'); + $pgbench->AddFile('src/bin/psql/psqlscan.l'); + $pgbench->AddIncludeDir('src/bin/psql'); + $solution->Save(); return $solution->{vcver}; } -- 1.8.3.1
>From 6e0deb626f8c32c38b8b9e08cfcf615c0e297c3a Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyot...@lab.ntt.co.jp> Date: Wed, 19 Aug 2015 12:53:13 +0900 Subject: [PATCH 4/4] Change the way to hold command list. Commands are generated as a linked list and stored into and accessed as an array. This patch unifies the way to store them to linked list. --- src/bin/pgbench/pgbench.c | 189 +++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 94 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index b6fd399..285ccca 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -191,16 +191,29 @@ typedef struct #define MAX_FILES 128 /* max number of SQL script files allowed */ #define SHELL_COMMAND_SIZE 256 /* maximum size allowed for shell command */ +#define MAX_ARGS 10 /* * structures used in custom query mode */ +typedef struct Command_t +{ + char *line; /* full text of command line */ + int command_num; /* unique index of this Command struct */ + int type; /* command type (SQL_COMMAND or META_COMMAND) */ + int argc; /* number of command words */ + char *argv[MAX_ARGS]; /* command word list */ + int cols[MAX_ARGS]; /* corresponding column starting from 1 */ + PgBenchExpr *expr; /* parsed expression */ + struct Command_t *next; /* more command if any, for multistatements */ +} Command; + typedef struct { PGconn *con; /* connection handle to DB */ int id; /* client No. */ - int state; /* state No. */ + Command *curr; /* current command */ int listen; /* 0 indicates that an async query has been * sent */ int sleeping; /* 1 indicates that the client is napping */ @@ -252,7 +265,6 @@ typedef struct */ #define SQL_COMMAND 1 #define META_COMMAND 2 -#define MAX_ARGS 10 typedef enum QueryMode { @@ -265,18 +277,6 @@ typedef enum QueryMode static QueryMode querymode = QUERY_SIMPLE; static const char *QUERYMODE[] = {"simple", "extended", "prepared"}; -typedef struct Command_t -{ - char *line; /* full text of command line */ - int command_num; /* unique index of this Command struct */ - int type; /* command type (SQL_COMMAND or META_COMMAND) */ - int argc; /* number of command words */ - char *argv[MAX_ARGS]; /* command word list */ - int cols[MAX_ARGS]; /* corresponding column starting from 1 */ - PgBenchExpr *expr; /* parsed expression */ - struct Command_t *next; /* more command if any, for multistatements */ -} Command; - typedef struct { @@ -312,7 +312,7 @@ typedef struct ParseInfo } ParseInfoData; typedef ParseInfoData *ParseInfo; -static Command **sql_files[MAX_FILES]; /* SQL script files */ +static Command *sql_files[MAX_FILES]; /* SQL script files */ static int num_files; /* number of script files */ static int num_commands = 0; /* total number of Command structs */ static int debug = 0; /* debug flag */ @@ -1140,12 +1140,27 @@ agg_vals_init(AggVals *aggs, instr_time start) aggs->start_time = INSTR_TIME_GET_DOUBLE(start); } +/* Return the ordinal of a command list item in a list */ +static int +get_command_number(Command *head, Command *curr) +{ + int i; + Command *p = head; + + for (i = 0 ; p && p != curr ; p = p->next, i++); + + /* curr must be in the list */ + Assert(p); + + return i; +} + /* return false iff client should be disconnected */ static bool doCustom(TState *thread, CState *st, instr_time *conn_time, FILE *logfile, AggVals *agg) { PGresult *res; - Command **commands; + Command *commands; bool trans_needs_throttle = false; instr_time now; @@ -1242,13 +1257,14 @@ top: if (st->listen) { /* are we receiver? */ - if (commands[st->state]->type == SQL_COMMAND) + if (st->curr->type == SQL_COMMAND) { if (debug) fprintf(stderr, "client %d receiving\n", st->id); if (!PQconsumeInput(st->con)) { /* there's something wrong */ - fprintf(stderr, "client %d aborted in state %d; perhaps the backend died while processing\n", st->id, st->state); + fprintf(stderr, "client %d aborted in state %d; perhaps the backend died while processing\n", + st->id, get_command_number(commands, st->curr)); return clientDone(st, false); } if (PQisBusy(st->con)) @@ -1261,7 +1277,7 @@ top: */ if (is_latencies) { - int cnum = commands[st->state]->command_num; + int cnum = st->curr->command_num; if (INSTR_TIME_IS_ZERO(now)) INSTR_TIME_SET_CURRENT(now); @@ -1271,7 +1287,7 @@ top: } /* transaction finished: calculate latency and log the transaction */ - if (commands[st->state + 1] == NULL) + if (st->curr->next == NULL) { /* only calculate latency if an option is used that needs it */ if (progress || throttle_delay || latency_limit) @@ -1304,7 +1320,7 @@ top: doLog(thread, st, logfile, &now, agg, false); } - if (commands[st->state]->type == SQL_COMMAND) + if (st->curr->type == SQL_COMMAND) { /* * Read and discard the query result; note this is not included in @@ -1318,7 +1334,8 @@ top: break; /* OK */ default: fprintf(stderr, "client %d aborted in state %d: %s", - st->id, st->state, PQerrorMessage(st->con)); + st->id, get_command_number(commands, st->curr), + PQerrorMessage(st->con)); PQclear(res); return clientDone(st, false); } @@ -1326,7 +1343,7 @@ top: discard_response(st); } - if (commands[st->state + 1] == NULL) + if (st->curr->next == NULL) { if (is_connect) { @@ -1340,12 +1357,12 @@ top: } /* increment state counter */ - st->state++; - if (commands[st->state] == NULL) + st->curr = st->curr->next; + if (st->curr == NULL) { - st->state = 0; st->use_file = (int) getrand(thread, 0, num_files - 1); commands = sql_files[st->use_file]; + st->curr = commands; st->is_throttled = false; /* @@ -1388,7 +1405,8 @@ top: } /* Record transaction start time under logging, progress or throttling */ - if ((logfile || progress || throttle_delay || latency_limit) && st->state == 0) + if ((logfile || progress || throttle_delay || latency_limit) && + st->curr == commands) { INSTR_TIME_SET_CURRENT(st->txn_begin); @@ -1404,9 +1422,9 @@ top: if (is_latencies) INSTR_TIME_SET_CURRENT(st->stmt_begin); - if (commands[st->state]->type == SQL_COMMAND) + if (st->curr->type == SQL_COMMAND) { - const Command *command = commands[st->state]; + const Command *command = st->curr; int r; if (querymode == QUERY_SIMPLE) @@ -1440,18 +1458,19 @@ top: if (!st->prepared[st->use_file]) { - int j; + int j = 0; + Command *pcom = commands; - for (j = 0; commands[j] != NULL; j++) + for (; pcom ; pcom = pcom->next, j++) { PGresult *res; char name[MAX_PREPARE_NAME]; - if (commands[j]->type != SQL_COMMAND) + if (pcom->type != SQL_COMMAND) continue; preparedStatementName(name, st->use_file, j); res = PQprepare(st->con, name, - commands[j]->argv[0], commands[j]->argc - 1, NULL); + pcom->argv[0], pcom->argc - 1, NULL); if (PQresultStatus(res) != PGRES_COMMAND_OK) fprintf(stderr, "%s", PQerrorMessage(st->con)); PQclear(res); @@ -1460,7 +1479,8 @@ top: } getQueryParams(st, command, params); - preparedStatementName(name, st->use_file, st->state); + preparedStatementName(name, st->use_file, + get_command_number(commands, st->curr)); if (debug) fprintf(stderr, "client %d sending %s\n", st->id, name); @@ -1480,11 +1500,11 @@ top: else st->listen = 1; /* flags that should be listened */ } - else if (commands[st->state]->type == META_COMMAND) + else if (st->curr->type == META_COMMAND) { - int argc = commands[st->state]->argc, + int argc = st->curr->argc, i; - char **argv = commands[st->state]->argv; + char **argv = st->curr->argv; if (debug) { @@ -1626,7 +1646,7 @@ top: else if (pg_strcasecmp(argv[0], "set") == 0) { char res[64]; - PgBenchExpr *expr = commands[st->state]->expr; + PgBenchExpr *expr = st->curr->expr; int64 result; if (!evaluateExpr(st, expr, &result)) @@ -2629,14 +2649,11 @@ read_line_from_file(FILE *fd) static int process_file(char *filename) { -#define COMMANDS_ALLOC_NUM 128 - - Command **my_commands; + Command *my_commands = NULL, + *my_commands_tail = NULL; FILE *fd; - int lineno, - index; + int lineno; char *buf; - int alloc_num; ParseInfo proc_state = createParseInfo(); if (num_files >= MAX_FILES) @@ -2645,23 +2662,18 @@ process_file(char *filename) exit(1); } - alloc_num = COMMANDS_ALLOC_NUM; - my_commands = (Command **) pg_malloc(sizeof(Command *) * alloc_num); - if (strcmp(filename, "-") == 0) fd = stdin; else if ((fd = fopen(filename, "r")) == NULL) { fprintf(stderr, "could not open file \"%s\": %s\n", filename, strerror(errno)); - pg_free(my_commands); return false; } proc_state->mode = PS_IDLE; lineno = 0; - index = 0; while ((buf = read_line_from_file(fd)) != NULL) { @@ -2677,52 +2689,42 @@ process_file(char *filename) /* * command is NULL when psql_scan returns PSCAN_EOL or * PSCAN_INCOMPLETE. Immediately ask for the next line for the - * cases. + * case. */ continue; } - while (command) - { - my_commands[index++] = command; - command = command->next; - } + /* Append new commands at the end of the list */ + if (my_commands_tail) + my_commands_tail->next = command; + else + my_commands = my_commands_tail = command; - if (index > alloc_num) - { - alloc_num += COMMANDS_ALLOC_NUM; - my_commands = pg_realloc(my_commands, - sizeof(Command *) * alloc_num); - } + /* Seek to the tail of the list */ + while (my_commands_tail->next) + my_commands_tail = my_commands_tail->next; } fclose(fd); parse_finish_scan(proc_state); - my_commands[index] = NULL; + my_commands_tail->next = NULL; sql_files[num_files++] = my_commands; return true; } -static Command ** +static Command * process_builtin(char *tb, const char *source) { -#define COMMANDS_ALLOC_NUM 128 - - Command **my_commands; - int lineno, - index; + Command *my_commands = NULL, + *my_commands_tail = NULL; + int lineno; char buf[BUFSIZ]; - int alloc_num; ParseInfo proc_state = createParseInfo(); - alloc_num = COMMANDS_ALLOC_NUM; - my_commands = (Command **) pg_malloc(sizeof(Command *) * alloc_num); - lineno = 0; - index = 0; for (;;) { @@ -2747,19 +2749,17 @@ process_builtin(char *tb, const char *source) if (command == NULL) continue; - /* builtin doesn't need multistatements */ + /* For simplisity, inhibit builtin from multistatements */ Assert(command->next == NULL); - my_commands[index] = command; - index++; - - if (index >= alloc_num) + if (my_commands_tail) { - alloc_num += COMMANDS_ALLOC_NUM; - my_commands = pg_realloc(my_commands, sizeof(Command *) * alloc_num); + my_commands_tail->next = command; + my_commands_tail = command; } + else + my_commands = my_commands_tail = command; } - my_commands[index] = NULL; parse_finish_scan(proc_state); return my_commands; @@ -2864,16 +2864,16 @@ printResults(int ttype, int64 normal_xacts, int nclients, for (i = 0; i < num_files; i++) { - Command **commands; + Command *command; if (num_files > 1) printf("statement latencies in milliseconds, file %d:\n", i + 1); else printf("statement latencies in milliseconds:\n"); - for (commands = sql_files[i]; *commands != NULL; commands++) + for (command = sql_files[i]; command ; + command=command->next) { - Command *command = *commands; int cnum = command->command_num; double total_time; instr_time total_exec_elapsed; @@ -3153,7 +3153,7 @@ main(int argc, char **argv) benchmarking_option_set = true; ttype = 3; filename = pg_strdup(optarg); - if (process_file(filename) == false || *sql_files[num_files - 1] == NULL) + if (process_file(filename) == false || sql_files[num_files - 1] == NULL) exit(1); break; case 'D': @@ -3735,17 +3735,19 @@ threadRun(void *arg) for (i = 0; i < nstate; i++) { CState *st = &state[i]; - Command **commands = sql_files[st->use_file]; + Command *commands = sql_files[st->use_file]; int prev_ecnt = st->ecnt; st->use_file = getrand(thread, 0, num_files - 1); + st->curr = sql_files[st->use_file]; + if (!doCustom(thread, st, &thread->conn_time, logfile, &aggs)) remains--; /* I've aborted */ - if (st->ecnt > prev_ecnt && commands[st->state]->type == META_COMMAND) + if (st->ecnt > prev_ecnt && st->curr->type == META_COMMAND) { fprintf(stderr, "client %d aborted in state %d; execution of meta-command failed\n", - i, st->state); + i, get_command_number(commands, st->curr)); remains--; /* I've aborted */ PQfinish(st->con); st->con = NULL; @@ -3766,7 +3768,6 @@ threadRun(void *arg) for (i = 0; i < nstate; i++) { CState *st = &state[i]; - Command **commands = sql_files[st->use_file]; int sock; if (st->con == NULL) @@ -3802,7 +3803,7 @@ threadRun(void *arg) min_usec = this_usec; } } - else if (commands[st->state]->type == META_COMMAND) + else if (st->curr->type == META_COMMAND) { min_usec = 0; /* the connection is ready to run */ break; @@ -3872,20 +3873,20 @@ threadRun(void *arg) for (i = 0; i < nstate; i++) { CState *st = &state[i]; - Command **commands = sql_files[st->use_file]; + Command *commands = sql_files[st->use_file]; int prev_ecnt = st->ecnt; if (st->con && (FD_ISSET(PQsocket(st->con), &input_mask) - || commands[st->state]->type == META_COMMAND)) + || st->curr->type == META_COMMAND)) { if (!doCustom(thread, st, &thread->conn_time, logfile, &aggs)) remains--; /* I've aborted */ } - if (st->ecnt > prev_ecnt && commands[st->state]->type == META_COMMAND) + if (st->ecnt > prev_ecnt && st->curr->type == META_COMMAND) { fprintf(stderr, "client %d aborted in state %d; execution of meta-command failed\n", - i, st->state); + i, get_command_number(commands, st->curr)); remains--; /* I've aborted */ PQfinish(st->con); st->con = NULL; -- 1.8.3.1
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers