Hi I am sending a initial implementation of psql content commands. This design is reaction to Tom's objections against psql file ref variables. This design is more cleaner, more explicit and more practical - import can be in one step.
Now supported commands are: r - read file without any modification rq - read file, escape as literal and use outer quotes rb - read binary file - transform to hex code string rbq - read binary file, transform to hex code string and use outer quotes Usage: create table testt(a xml); insert into test values( {rq /home/pavel/.local/share/rhythmbox/rhythmdb.xml} ); \set xxx {r /home/pavel/.local/share/rhythmbox/rhythmdb.xml} This patch is demo of this design - one part is redundant - I'll clean it in next iteration. Regards Pavel
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index a7789df..dbf3ffa 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -12,6 +12,7 @@ #include <limits.h> #include <math.h> #include <signal.h> +#include <sys/stat.h> #ifndef WIN32 #include <unistd.h> /* for write() */ #else @@ -168,6 +169,142 @@ psql_get_variable(const char *varname, bool escape, bool as_ident) return result; } +/* + * file-content-fetching callback for flex lexer. + */ +char * +psql_get_file_content(const char *filename, bool escape, bool binary, bool quote) +{ + FILE *fd; + char *fname = pg_strdup(filename); + char *result = NULL; + + expand_tilde(&fname); + canonicalize_path(fname); + + fd = fopen(fname, PG_BINARY_R); + if (fd) + { + struct stat fst; + + if (fstat(fileno(fd), &fst) != -1) + { + if (S_ISREG(fst.st_mode)) + { + if (fst.st_size <= ((int64) 1024) * 1024 * 1024) + { + size_t size; + PQExpBufferData raw_data; + char buf[512]; + + initPQExpBuffer(&raw_data); + + if (!escape && quote) + appendPQExpBufferChar(&raw_data, '\''); + + while ((size = fread(buf, 1, sizeof(buf), fd)) > 0) + appendBinaryPQExpBuffer(&raw_data, buf, size); + + if (!ferror(fd) && !(PQExpBufferDataBroken(raw_data))) + { + if (escape) + { + if (binary) + { + unsigned char *escaped_value; + size_t escaped_size; + + escaped_value = PQescapeByteaConn(pset.db, + (const unsigned char *) raw_data.data, raw_data.len, + &escaped_size); + + if (escaped_value != NULL) + { + if (quote) + { + PQExpBufferData finalbuf; + + initPQExpBuffer(&finalbuf); + appendPQExpBufferChar(&finalbuf, '\''); + appendBinaryPQExpBuffer(&finalbuf, + (const char *) escaped_value, escaped_size - 1); + appendPQExpBufferChar(&finalbuf, '\''); + PQfreemem(escaped_value); + + result = finalbuf.data; + } + else + result = (char *) escaped_value; + } + else + psql_error("%s\n", PQerrorMessage(pset.db)); + } + else + { + /* escape text */ + if (quote) + { + result = PQescapeLiteral(pset.db, + raw_data.data, raw_data.len); + if (result == NULL) + psql_error("%s\n", PQerrorMessage(pset.db)); + } + else + { + int error; + + result = pg_malloc(raw_data.len * 2 + 1); + PQescapeStringConn(pset.db, result, raw_data.data, raw_data.len, &error); + if (error) + { + psql_error("%s\n", PQerrorMessage(pset.db)); + PQfreemem(result); + result = NULL; + } + } + } + } + else + { + /* returns raw data, without any transformations */ + if (quote) + appendPQExpBufferChar(&raw_data, '\''); + + if (PQExpBufferDataBroken(raw_data)) + psql_error("out of memory\n"); + else + result = raw_data.data; + } + } + else + { + if (PQExpBufferDataBroken(raw_data)) + psql_error("out of memory\n"); + else + psql_error("%s: %s\n", fname, strerror(errno)); + } + + if (result != raw_data.data) + termPQExpBuffer(&raw_data); + } + else + psql_error("%s is too big (greather than 1GB)\n", fname); + } + else + psql_error("%s is not regular file\n", fname); + } + else + psql_error("%s: %s\n", fname, strerror(errno)); + + fclose(fd); + } + else + psql_error("%s: %s\n", fname, strerror(errno)); + + PQfreemem(fname); + + return result; +} /* * Error reporting for scripts. Errors should look like diff --git a/src/bin/psql/common.h b/src/bin/psql/common.h index bdcb58f..6b8ae67 100644 --- a/src/bin/psql/common.h +++ b/src/bin/psql/common.h @@ -19,6 +19,7 @@ extern bool openQueryOutputFile(const char *fname, FILE **fout, bool *is_pipe); extern bool setQFout(const char *fname); extern char *psql_get_variable(const char *varname, bool escape, bool as_ident); +extern char *psql_get_file_content(const char *filename, bool escape, bool binary, bool quote); extern void psql_error(const char *fmt,...) pg_attribute_printf(1, 2); diff --git a/src/bin/psql/mainloop.c b/src/bin/psql/mainloop.c index 37dfa4d..bf1d89f 100644 --- a/src/bin/psql/mainloop.c +++ b/src/bin/psql/mainloop.c @@ -20,6 +20,7 @@ /* callback functions for our flex lexer */ const PsqlScanCallbacks psqlscan_callbacks = { psql_get_variable, + psql_get_file_content, psql_error }; diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index 86832a8..1b1080c 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -19,6 +19,7 @@ #include "postgres_fe.h" #include "psqlscanslash.h" +#include "common.h" #include "libpq-fe.h" } @@ -46,6 +47,7 @@ static enum slash_option_type option_type; static char *option_quote; static int unquoted_option_chars; static int backtick_start_offset; +static int curlybracket_start_offset; /* Return values from yylex() */ @@ -54,6 +56,7 @@ static int backtick_start_offset; static void evaluate_backtick(PsqlScanState state); +static void evaluate_curlybrackets(PsqlScanState state); #define ECHO psqlscan_emit(cur_state, yytext, yyleng) @@ -98,6 +101,7 @@ extern void slash_yyset_column(int column_no, yyscan_t yyscanner); %x xslashdquote %x xslashwholeline %x xslashend +%x xslashcurlybrackets /* * Assorted character class definitions that should match psqlscan.l. @@ -228,6 +232,14 @@ other . BEGIN(xslashdquote); } +"{" { + curlybracket_start_offset = output_buf->len; + ECHO; + *option_quote = '{'; + unquoted_option_chars = 0; + BEGIN(xslashcurlybrackets); + } + :{variable_char}+ { /* Possible psql variable substitution */ if (option_type == OT_NO_EVAL || @@ -362,6 +374,21 @@ other . } +<xslashcurlybrackets>{ + /* curly brackets: copy everything until next right curly bracket */ + +"}" { + /* In NO_EVAL mode, don't evaluate the command */ + ECHO; + if (option_type != OT_NO_EVAL) + evaluate_curlybrackets(cur_state); + BEGIN(xslasharg); + } + +{other}|\n { ECHO; } + +} + <xslashdquote>{ /* double-quoted text: copy verbatim, including the double quotes */ @@ -580,6 +607,7 @@ psql_scan_slash_option(PsqlScanState state, case xslashquote: case xslashbackquote: case xslashdquote: + case xslashcurlybrackets: /* must have hit EOL inside quotes */ state->callbacks->write_error("unterminated quoted string\n"); termPQExpBuffer(&mybuf); @@ -755,3 +783,107 @@ evaluate_backtick(PsqlScanState state) termPQExpBuffer(&cmd_output); } + +static void +evaluate_curlybrackets(PsqlScanState state) +{ + PQExpBuffer output_buf = state->output_buf; + char *cmdline = output_buf->data + curlybracket_start_offset; + char *cmdlinebuf; + char *endptr; + bool read_file = false; + bool binary = false; + bool escape = false; + bool quote = false; + + cmdlinebuf = pg_strdup(output_buf->data + curlybracket_start_offset); + + /* skip initial left bracket */ + cmdline = cmdlinebuf + 1; + + /* skip initial spaces */ + while (*cmdline == ' ') + cmdline++; + + /* we should to remove final right bracket and trim spaces */ + endptr = cmdline + strlen(cmdline); + *(--endptr) = '\0'; + endptr--; + while (*endptr == ' ' && endptr > cmdline) + endptr--; + + endptr[1] = '\0'; + + if (*cmdline != '\0') + { + char *cptr = cmdline; + int clen; + char cname[10]; + + /* find a end of statement */ + while (*cptr != ' ' && *cptr != '\0') + cptr++; + + clen = cptr - cmdline; + if (clen < 10) + { + strncpy(cname, cmdline, clen); + cname[clen] = '\0'; + + if (strcmp(cname, "r") == 0) + { + read_file = true; + } + else if (strcmp(cname, "rb") == 0) + { + read_file = true; + binary = true; + escape = true; + } + else if (strcmp(cname, "rq") == 0) + { + read_file = true; + escape = true; + quote = true; + } + else if (strcmp(cname, "rbq") == 0) + { + read_file = true; + escape = true; + binary = true; + quote = true; + } + else + state->callbacks->write_error("%s: unsupported psql content command", cname); + + if (read_file) + { + /* skip initial spaces */ + while (*cptr == ' ') + cptr++; + + if (cptr != '\0') + { + char *content = psql_get_file_content(cptr, escape, binary, quote); + + if (content != NULL) + { + output_buf->len = curlybracket_start_offset; + output_buf->data[output_buf->len] = '\0'; + + appendPQExpBufferStr(output_buf, content); + PQfreemem(content); + } + } + else + state->callbacks->write_error("%s: missing expected file name", cname); + } + } + else + state->callbacks->write_error("%s: psql content command is too long", cmdline); + } + else + state->callbacks->write_error("empty psql content command"); + + PQfreemem(cmdlinebuf); +} diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index b556c00..b8d6c83 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -58,7 +58,7 @@ extern char *filename_completion_function(); #endif /* word break characters */ -#define WORD_BREAKS "\t\n@$><=;|&{() " +#define WORD_BREAKS "\t\n@$><=;|&() " /* * Since readline doesn't let us pass any state through to the tab completion @@ -1343,6 +1343,11 @@ psql_completion(const char *text, int start, int end) "\\timing", "\\unset", "\\x", "\\w", "\\watch", "\\z", "\\!", NULL }; + /* psql's content commands. */ + static const char *const content_commands[] = { + "{r", "{rq", "{rb", "{rbq", NULL + }; + (void) end; /* "end" is not used */ #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER @@ -1368,6 +1373,10 @@ psql_completion(const char *text, int start, int end) if (text[0] == '\\') COMPLETE_WITH_LIST_CS(backslash_commands); + else if (text[0] == '{') + { + COMPLETE_WITH_LIST_CS(content_commands); + } /* If current word is a variable interpolation, handle that case */ else if (text[0] == ':' && text[1] != ':') { @@ -3222,6 +3231,11 @@ psql_completion(const char *text, int start, int end) completion_charp = "\\"; matches = completion_matches(text, complete_from_files); } + else if (TailMatchesCS1("\{r|\{rb|\{rq|\{rbq")) + { + completion_charp = "{"; + matches = completion_matches(text, complete_from_files); + } /* * Finally, we look through the list of "things", such as TABLE, INDEX and diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l index 55067b4..1bfbf12 100644 --- a/src/fe_utils/psqlscan.l +++ b/src/fe_utils/psqlscan.l @@ -47,6 +47,8 @@ */ typedef int YYSTYPE; +static int curlybracket_start_offset; + /* * Set the type of yyextra; we use it as a pointer back to the containing * PsqlScanState. @@ -71,6 +73,8 @@ typedef int YYSTYPE; extern int psql_yyget_column(yyscan_t yyscanner); extern void psql_yyset_column(int column_no, yyscan_t yyscanner); +static void evaluate_curlybrackets(PsqlScanState state, int start_offset); + %} %option reentrant @@ -115,6 +119,7 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner); * <xuiend> end of a quoted identifier with Unicode escapes, UESCAPE can follow * <xus> quoted string with Unicode escapes * <xusend> end of a quoted string with Unicode escapes, UESCAPE can follow + * <xcb> curly bracket string * * Note: we intentionally don't mimic the backend's <xeu> state; we have * no need to distinguish it from <xe> state, and no good way to get out @@ -133,6 +138,7 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner); %x xuiend %x xus %x xusend +%x xcb /* * In order to make the world safe for Windows and Mac clients as well as @@ -673,6 +679,22 @@ other . } } +"{" { + curlybracket_start_offset = cur_state->output_buf->len; + BEGIN(xcb); + ECHO; + } + +<xcb>"}" { + ECHO; + evaluate_curlybrackets(cur_state, curlybracket_start_offset); + BEGIN(INITIAL); + } + +<xcb>. { + ECHO; + } + /* * psql-specific rules to handle backslash commands and variable * substitution. We want these before {self}, also. @@ -1426,3 +1448,114 @@ psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, psqlscan_emit(state, txt, len); } } + +/* + * evaluate a content command in curly brackets + */ +static void +evaluate_curlybrackets(PsqlScanState state, int start_offset) +{ + PQExpBuffer output_buf = state->output_buf; + char *cmdline; + char *cmdlinebuf; + char *endptr; + bool read_file = false; + bool binary = false; + bool escape = false; + bool quote = false; + + cmdlinebuf = pg_strdup(output_buf->data + start_offset); + + /* skip initial left bracket */ + cmdline = cmdlinebuf + 1; + + /* skip initial spaces */ + while (*cmdline == ' ') + cmdline++; + + /* we should to remove final right bracket, and trim spaces */ + endptr = cmdline + strlen(cmdline); + *(--endptr) = '\0'; + endptr--; + while (*endptr == ' ' && endptr > cmdline) + endptr--; + + endptr[1] = '\0'; + + if (*cmdline != '\0') + { + char *cptr = cmdline; + int clen; + char cname[10]; + + /* find a end of statement */ + while (*cptr != ' ' && *cptr != '\0') + cptr++; + + clen = cptr - cmdline; + if (clen < 10) + { + strncpy(cname, cmdline, clen); + cname[clen] = '\0'; + + if (strcmp(cname, "r") == 0) + { + read_file = true; + } + else if (strcmp(cname, "rb") == 0) + { + read_file = true; + binary = true; + escape = true; + } + else if (strcmp(cname, "rq") == 0) + { + read_file = true; + escape = true; + quote = true; + } + else if (strcmp(cname, "rbq") == 0) + { + read_file = true; + escape = true; + binary = true; + quote = true; + } + else + state->callbacks->write_error("%s: unsupported psql content command\n", cname); + + if (read_file) + { + /* skip initial spaces */ + while (*cptr == ' ') + cptr++; + + if (cptr != '\0') + { + if (state->callbacks->get_file_content) + { + char *content = state->callbacks->get_file_content(cptr, + escape, binary, quote); + + if (content != NULL) + { + output_buf->len = start_offset; + output_buf->data[output_buf->len] = '\0'; + + appendPQExpBufferStr(output_buf, content); + PQfreemem(content); + } + } + } + else + state->callbacks->write_error("%s: missing expected file name\n", cname); + } + } + else + state->callbacks->write_error("%s: psql content command is too long\n", cmdline); + } + else + state->callbacks->write_error("empty psql content command\n"); + + PQfreemem(cmdlinebuf); +} diff --git a/src/include/fe_utils/psqlscan.h b/src/include/fe_utils/psqlscan.h index 1f10ecc..93dfa5e 100644 --- a/src/include/fe_utils/psqlscan.h +++ b/src/include/fe_utils/psqlscan.h @@ -54,6 +54,8 @@ typedef struct PsqlScanCallbacks /* Fetch value of a variable, as a pfree'able string; NULL if unknown */ /* This pointer can be NULL if no variable substitution is wanted */ char *(*get_variable) (const char *varname, bool escape, bool as_ident); + /* Fetch content of a file, as a pfree'able string */ + char *(*get_file_content) (const char *filename, bool escape, bool binary, bool quote); /* Print an error message someplace appropriate */ /* (very old gcc versions don't support attributes on function pointers) */ #if defined(__GNUC__) && __GNUC__ < 4
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers