Hi, On 2022-10-01 23:56:59 -0700, Andres Freund wrote: > On 2022-09-12 09:58:37 +0200, Daniel Gustafsson wrote: > > Correct, fixed in the attached. > > Updated patch adding meson compatibility attached.
Err, forgot to amend one hunk :( Greetings, Andres Freund
>From fe2926ccd49a460cbaa39a8916a4dd097463b294 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <dan...@yesql.se> Date: Wed, 7 Sep 2022 15:22:05 +0200 Subject: [PATCH v6] Add include/exclude filtering via file in pg_dump Author: Pavel Stehule <pavel.steh...@gmail.com> Discussion: https://postgr.es/m/CAFj8pRB10wvW0CC9Xq=1XDs=zcqxer3cblcnza+qix4cuh-...@mail.gmail.com --- src/bin/pg_dump/.gitignore | 4 + src/bin/pg_dump/Makefile | 17 +++- src/bin/pg_dump/filter.h | 44 ++++++++++ src/bin/pg_dump/filterparse.y | 64 ++++++++++++++ src/bin/pg_dump/filterscan.l | 139 ++++++++++++++++++++++++++++++ src/bin/pg_dump/meson.build | 18 ++++ src/bin/pg_dump/pg_backup_utils.c | 33 +++++++ src/bin/pg_dump/pg_backup_utils.h | 1 + src/bin/pg_dump/pg_dump.c | 56 ++++++++++++ src/bin/pg_dump/pg_dumpall.c | 65 +++++++++++++- src/bin/pg_dump/pg_restore.c | 58 +++++++++++++ doc/src/sgml/ref/pg_dump.sgml | 88 +++++++++++++++++++ doc/src/sgml/ref/pg_dumpall.sgml | 22 +++++ doc/src/sgml/ref/pg_restore.sgml | 25 ++++++ src/tools/msvc/Mkvcbuild.pm | 4 +- 15 files changed, 634 insertions(+), 4 deletions(-) create mode 100644 src/bin/pg_dump/filter.h create mode 100644 src/bin/pg_dump/filterparse.y create mode 100644 src/bin/pg_dump/filterscan.l diff --git a/src/bin/pg_dump/.gitignore b/src/bin/pg_dump/.gitignore index e6d78127793..11f2d68bea0 100644 --- a/src/bin/pg_dump/.gitignore +++ b/src/bin/pg_dump/.gitignore @@ -2,4 +2,8 @@ /pg_dumpall /pg_restore +# Local generated source files +/filterparse.c +/filterscan.c + /tmp_check/ diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 9dc5a784dd2..e3befdc9b1f 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -26,6 +26,8 @@ OBJS = \ $(WIN32RES) \ compress_io.o \ dumputils.o \ + filterparse.o \ + filterscan.o \ parallel.o \ pg_backup_archiver.o \ pg_backup_custom.o \ @@ -37,14 +39,23 @@ OBJS = \ all: pg_dump pg_restore pg_dumpall +# See notes in src/backend/parser/Makefile about the following two rules +filterparse.h: filterparse.c + touch $@ + +filterparse.c: BISONFLAGS += -d + +# Force these dependencies to be known even without dependency info built: +filterparse.o filterscan.o: filterparse.h + pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) pg_dump.o common.o pg_dump_sort.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils - $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_dumpall: pg_dumpall.o dumputils.o filterparse.o filterscan.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filterparse.o filterscan.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) @@ -63,6 +74,8 @@ installcheck: uninstall: rm -f $(addprefix '$(DESTDIR)$(bindir)'/, pg_dump$(X) pg_restore$(X) pg_dumpall$(X)) +distprep: filterparse.c filterscan.c + clean distclean maintainer-clean: rm -f pg_dump$(X) pg_restore$(X) pg_dumpall$(X) $(OBJS) pg_dump.o common.o pg_dump_sort.o pg_restore.o pg_dumpall.o rm -rf tmp_check diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h new file mode 100644 index 00000000000..5dff4161f02 --- /dev/null +++ b/src/bin/pg_dump/filter.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * filter.h + * Common header file for the parser of filter file + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/bin/pg_dump/filter.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILTER_H +#define FILTER_H +#include "c.h" + +/* + * List of objects that can be specified in filter file + */ +typedef enum +{ + FILTER_OBJECT_TYPE_NONE, + FILTER_OBJECT_TYPE_DATA, + FILTER_OBJECT_TYPE_DATABASE, + FILTER_OBJECT_TYPE_FOREIGN_DATA, + FILTER_OBJECT_TYPE_FUNCTION, + FILTER_OBJECT_TYPE_INDEX, + FILTER_OBJECT_TYPE_SCHEMA, + FILTER_OBJECT_TYPE_TABLE, + FILTER_OBJECT_TYPE_TRIGGER, +} FilterObjectType; + +extern FILE *filter_yyin; + +extern int filter_yylex(void); +extern void filter_yyerror(void *priv, const char *msg); +extern void filter_scanner_init(void); +extern void filter_scanner_finish(void); +extern int filter_yyparse(void *priv); + +extern void include_item(void *priv, FilterObjectType objtype, const char *objname); +extern void exclude_item(void *priv, FilterObjectType objtype, const char *objname); + +#endif diff --git a/src/bin/pg_dump/filterparse.y b/src/bin/pg_dump/filterparse.y new file mode 100644 index 00000000000..c7f06f788d3 --- /dev/null +++ b/src/bin/pg_dump/filterparse.y @@ -0,0 +1,64 @@ +%{ +/*------------------------------------------------------------------------- + * + * filterparse.y + * bison grammar for the pg_dump object filter files + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/bin/pg_dump/filterparse.y + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "filter.h" + +%} + +%name-prefix="filter_yy" +%parse-param {void *priv} + +%union +{ + char *str; + int integer; +} + +%type <integer> include_object exclude_object +%token <str> pattern +%token C_INCLUDE C_EXCLUDE +%token OBJ_DATA OBJ_DATABASE OBJ_FOREIGN_DATA OBJ_FUNCTION OBJ_INDEX +%token OBJ_SCHEMA OBJ_TABLE OBJ_TRIGGER +%start Filters + +%% + +Filters: + /* EMPTY */ + | Filters Filter + ; + +Filter: + C_INCLUDE include_object pattern { include_item(priv, $2, $3); } + | C_EXCLUDE exclude_object pattern { exclude_item(priv, $2, $3); } + ; + +include_object: + OBJ_FOREIGN_DATA { $$ = FILTER_OBJECT_TYPE_FOREIGN_DATA; } + | OBJ_FUNCTION { $$ = FILTER_OBJECT_TYPE_FUNCTION; } + | OBJ_INDEX { $$ = FILTER_OBJECT_TYPE_INDEX; } + | OBJ_SCHEMA { $$ = FILTER_OBJECT_TYPE_SCHEMA; } + | OBJ_TABLE { $$ = FILTER_OBJECT_TYPE_TABLE; } + | OBJ_TRIGGER { $$ = FILTER_OBJECT_TYPE_TRIGGER; } + ; + +exclude_object: + OBJ_DATA { $$ = FILTER_OBJECT_TYPE_DATA; } + | OBJ_DATABASE { $$ = FILTER_OBJECT_TYPE_DATABASE; } + | OBJ_SCHEMA { $$ = FILTER_OBJECT_TYPE_SCHEMA; } + | OBJ_TABLE { $$ = FILTER_OBJECT_TYPE_TABLE; } + ; + +%% diff --git a/src/bin/pg_dump/filterscan.l b/src/bin/pg_dump/filterscan.l new file mode 100644 index 00000000000..0d7341086ba --- /dev/null +++ b/src/bin/pg_dump/filterscan.l @@ -0,0 +1,139 @@ +%top{ +/*------------------------------------------------------------------------- + * + * filterscan.l + * a lexical scanner for pg_dump object filter files + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/bin/pg_dump/filterscan.l + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" +#include "filterparse.h" +#include "filter.h" +} + +%{ + +static int yyline = 1; + +static void add_pattern_char(char c); + +#define PATTERNBUF_INIT 128 + +static char *patternbuf = NULL; +static size_t patternbufsize = PATTERNBUF_INIT; +static size_t patternbufpos = 0; + +%} + +%option never-interactive +%option noinput +%option nounput +%option noyywrap +%option warn +%option prefix="filter_yy" + +%x command +%x type +%x qident + +space [ \t\r\f\v] +newline [\n] +non_newline [^\n\r] + +comment ("#"{non_newline}*) +pattern ([^"\n# \t\r\f\v]+) + +%% + + /* Commands */ +include { BEGIN(command); return C_INCLUDE; } +exclude { BEGIN(command); return C_EXCLUDE; } + +[\n] yyline++; +{comment} { /* ignore */ } +{space} { /* ignore */ } + + /* Object types */ +<command>({space}*)data { BEGIN(type); return OBJ_DATA; } +<command>({space}*)database { BEGIN(type); return OBJ_DATABASE; } +<command>({space}*)foreign_data { BEGIN(type); return OBJ_FOREIGN_DATA; } +<command>({space}*)function { BEGIN(type); return OBJ_FUNCTION; } +<command>({space}*)index { BEGIN(type); return OBJ_INDEX; } +<command>({space}*)schema { BEGIN(type); return OBJ_SCHEMA; } +<command>({space}*)table { BEGIN(type); return OBJ_TABLE; } +<command>({space}*)trigger { BEGIN(type); return OBJ_TRIGGER; } +<command>{pattern} { filter_yyerror(NULL, "unsupported filter object type"); } +<command><<EOF>> { filter_yyerror(NULL, "missing type name"); } + + /* Patterns */ +<type><<EOF>> { filter_yyerror(NULL, "missing object name"); } +<type>{space} { /* ignore */ } +<type>{pattern} { + filter_yylval.str = pg_strdup(yytext); + BEGIN(INITIAL); + return(pattern); + } +<type>\" { + patternbufpos = 0; + memset(patternbuf, 0, patternbufsize); + BEGIN(qident); + } +<qident>\\\" { add_pattern_char(yytext[1]); } +<qident>\" { + filter_yylval.str = pg_strdup(patternbuf); + BEGIN(INITIAL); + return(pattern); + } +<qident>{newline} { add_pattern_char('\n'); } +<qident>. { add_pattern_char(yytext[0]); } +<qident><<EOF>> { + fprintf(stderr, "syntax error at line %i; unterminated quoted pattern: %s\n", + yyline, patternbuf); + exit(1); + } + /* Anything else is a syntax error */ +. { filter_yyerror(NULL, "unexpected character"); } +%% + +static void +add_pattern_char(char c) +{ + /* Leave room for trailing zero */ + if (patternbufpos >= patternbufsize - 1) + { + /* Ensure that we don't overflow */ + if (patternbufsize > (SIZE_MAX / 2)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } + + /* Double the size of patternbuf if it gets full */ + patternbufsize += patternbufsize; + patternbuf = pg_realloc(patternbuf, patternbufsize); + } + patternbuf[patternbufpos++] = c; +} + +void +filter_scanner_init(void) +{ + patternbuf = pg_malloc(patternbufsize); +} + +void +filter_scanner_finish(void) +{ + pfree(patternbuf); +} + +void +filter_yyerror(void *priv, const char *msg) +{ + fprintf(stderr, "syntax error at line %i: %s: \"%s\"\n", yyline, msg, yytext); + exit(1); +} diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build index 785ec094dbd..8ec15cd4534 100644 --- a/src/bin/pg_dump/meson.build +++ b/src/bin/pg_dump/meson.build @@ -11,8 +11,26 @@ pg_dump_common_sources = files( 'pg_backup_utils.c', ) + +filterscan = custom_target('filterscan', + input: 'filterscan.l', + output: 'filterscan.c', + command: flex_cmd, +) +generated_sources += filterscan +pg_dump_common_sources += filterscan + +filterparse = custom_target('filterparse', + input: 'filterparse.y', + output: 'filterparse.c', + command: bison_cmd, +) +generated_sources += filterparse +pg_dump_common_sources += filterparse + pg_dump_common = static_library('libpgdump_common', pg_dump_common_sources, + include_directories: '.', dependencies: [frontend_code, libpq, zlib], kwargs: internal_lib_args, ) diff --git a/src/bin/pg_dump/pg_backup_utils.c b/src/bin/pg_dump/pg_backup_utils.c index e40890cb264..d6db156bae8 100644 --- a/src/bin/pg_dump/pg_backup_utils.c +++ b/src/bin/pg_dump/pg_backup_utils.c @@ -13,6 +13,7 @@ */ #include "postgres_fe.h" +#include "filter.h" #include "parallel.h" #include "pg_backup_utils.h" @@ -102,3 +103,35 @@ exit_nicely(int code) exit(code); } + +/* + * Retrieve object identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + */ +void +parse_filters_from_file(const char *filename, void *priv) +{ + bool need_close = true; + + if (strlen(filename) == 1 && filename[0] == '-') + { + filter_yyin = stdin; + need_close = false; + } + else + { + filter_yyin = fopen(filename, "r"); + if (!filter_yyin) + pg_fatal("unable to open filterfile \"%s\": %m", filename); + } + + filter_scanner_init(); + filter_yyparse(priv); + filter_scanner_finish(); + + if (need_close) + fclose(filter_yyin); +} diff --git a/src/bin/pg_dump/pg_backup_utils.h b/src/bin/pg_dump/pg_backup_utils.h index 8173bb93cfc..74b942e712c 100644 --- a/src/bin/pg_dump/pg_backup_utils.h +++ b/src/bin/pg_dump/pg_backup_utils.h @@ -30,6 +30,7 @@ extern const char *progname; extern void set_dump_section(const char *arg, int *dumpSections); extern void on_exit_nicely(on_exit_nicely_callback function, void *arg); extern void exit_nicely(int code) pg_attribute_noreturn(); +extern void parse_filters_from_file(const char *filename, void *priv); /* In pg_dump, we modify pg_fatal to call exit_nicely instead of exit */ #undef pg_fatal diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index bd9b066e4eb..da69cee0cf1 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "dumputils.h" #include "fe_utils/option_utils.h" #include "fe_utils/string_utils.h" +#include "filter.h" #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" @@ -390,6 +391,7 @@ main(int argc, char **argv) {"enable-row-security", no_argument, &dopt.enable_row_security, 1}, {"exclude-table-data", required_argument, NULL, 4}, {"extra-float-digits", required_argument, NULL, 8}, + {"filter", required_argument, NULL, 12}, {"if-exists", no_argument, &dopt.if_exists, 1}, {"inserts", no_argument, NULL, 9}, {"lock-wait-timeout", required_argument, NULL, 2}, @@ -623,6 +625,10 @@ main(int argc, char **argv) optarg); break; + case 12: /* object filters from file */ + parse_filters_from_file(optarg, (void *) &dopt); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -1028,6 +1034,8 @@ help(const char *progname) " access to)\n")); printf(_(" --exclude-table-data=PATTERN do NOT dump data for the specified table(s)\n")); printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); + printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n" + " in specified file\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --include-foreign-data=PATTERN\n" " include data of foreign tables on foreign\n" @@ -18198,3 +18206,51 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, if (!res) pg_log_warning("could not parse %s array", "reloptions"); } + +void +exclude_item(void *priv, FilterObjectType objtype, const char *objname) +{ + (void) priv; + + switch (objtype) + { + case FILTER_OBJECT_TYPE_DATA: + simple_string_list_append(&tabledata_exclude_patterns, objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&schema_exclude_patterns, objname); + break; + case FILTER_OBJECT_TYPE_TABLE: + simple_string_list_append(&table_exclude_patterns, objname); + break; + default: + pg_log_error("Unsupported exclude object"); + exit_nicely(1); + break; + } +} + +void +include_item(void *priv, FilterObjectType objtype, const char *objname) +{ + DumpOptions *dopt = (DumpOptions *) priv; + + switch (objtype) + { + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + simple_string_list_append(&foreign_servers_include_patterns, objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&schema_include_patterns, objname); + dopt->include_everything = false; + break; + case FILTER_OBJECT_TYPE_TABLE: + simple_string_list_append(&table_include_patterns, objname); + dopt->include_everything = false; + break; + default: + pg_log_error("Unsupported include object"); + exit_nicely(1); + break; + } +} diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 083012ca39d..e2b0e241fd2 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -26,6 +26,7 @@ #include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" +#include "filter.h" #include "getopt_long.h" #include "pg_backup.h" @@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query); static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); +static void getDatabaseExcludeFiltersFromFile(const char *filename, SimpleStringList *patterns); static char pg_dump_bin[MAXPGPATH]; static const char *progname; @@ -158,6 +160,7 @@ main(int argc, char *argv[]) {"disable-triggers", no_argument, &disable_triggers, 1}, {"exclude-database", required_argument, NULL, 6}, {"extra-float-digits", required_argument, NULL, 5}, + {"filter", required_argument, NULL, 8}, {"if-exists", no_argument, &if_exists, 1}, {"inserts", no_argument, &inserts, 1}, {"lock-wait-timeout", required_argument, NULL, 2}, @@ -360,6 +363,11 @@ main(int argc, char *argv[]) appendShellString(pgdumpopts, optarg); break; + case 8: + getDatabaseExcludeFiltersFromFile(optarg, + &database_exclude_patterns); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -653,6 +661,7 @@ help(void) printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n")); printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); + printf(_(" --filter=FILENAME exclude databases specified in filter file\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --load-via-partition-root load partitions via the root table\n")); @@ -1890,7 +1899,6 @@ executeCommand(PGconn *conn, const char *query) PQclear(res); } - /* * dumpTimestamp */ @@ -1914,3 +1922,58 @@ hash_string_pointer(char *s) return hash_bytes(ss, strlen(s)); } + +/* + * getDatabaseFiltersFromFile - retrieve database identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + */ +static void +getDatabaseExcludeFiltersFromFile(const char *filename, SimpleStringList *pattern) +{ + bool need_close = true; + + if (strlen(filename) == 1 && filename[0] == '-') + { + filter_yyin = stdin; + need_close = false; + } + else + { + filter_yyin = fopen(filename, "r"); + if (!filter_yyin) + pg_fatal("unable to open filterfile \"%s\": %m", filename); + } + + filter_scanner_init(); + filter_yyparse((void *) pattern); + filter_scanner_finish(); + + if (need_close) + fclose(filter_yyin); +} + +void +exclude_item(void *priv, FilterObjectType objtype, const char *objname) +{ + SimpleStringList *pattern = (SimpleStringList *) priv; + + if (objtype != FILTER_OBJECT_TYPE_DATABASE) + { + pg_log_error("Unsupported exclude object"); + exit_nicely(1); + } + + simple_string_list_append(pattern, objname); +} + +void +include_item(void *priv, FilterObjectType objtype, const char *objname) +{ + (void) priv; + + pg_log_error("Unsupported include object"); + exit_nicely(1); +} diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 049a1006347..70897d90566 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -47,6 +47,7 @@ #include "dumputils.h" #include "fe_utils/option_utils.h" +#include "filter.h" #include "getopt_long.h" #include "parallel.h" #include "pg_backup_utils.h" @@ -123,6 +124,7 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"filter", required_argument, NULL, 4}, {NULL, 0, NULL, 0} }; @@ -286,6 +288,10 @@ main(int argc, char **argv) set_dump_section(optarg, &(opts->dumpSections)); break; + case 4: + parse_filters_from_file(optarg, (void *) opts); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -463,6 +469,7 @@ usage(const char *progname) printf(_(" -1, --single-transaction restore as a single transaction\n")); printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --enable-row-security enable row security\n")); + printf(_(" --filter=FILE restore objects based on filter expressions\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --no-comments do not restore comments\n")); printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n" @@ -494,3 +501,54 @@ usage(const char *progname) printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); } + +void +exclude_item(void *priv, FilterObjectType objtype, const char *objname) +{ + RestoreOptions *opts = (RestoreOptions *) priv; + + if (objtype != FILTER_OBJECT_TYPE_SCHEMA) + { + pg_log_error("Unsupported exclude object"); + exit_nicely(1); + } + + simple_string_list_append(&opts->schemaExcludeNames, objname); +} + +void +include_item(void *priv, FilterObjectType objtype, const char *objname) +{ + RestoreOptions *opts = (RestoreOptions *) priv; + + switch (objtype) + { + case FILTER_OBJECT_TYPE_FUNCTION: + opts->selTypes = 1; + opts->selFunction = 1; + simple_string_list_append(&opts->functionNames, objname); + break; + case FILTER_OBJECT_TYPE_INDEX: + opts->selTypes = 1; + opts->selIndex = 1; + simple_string_list_append(&opts->indexNames, objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&opts->schemaNames, objname); + break; + case FILTER_OBJECT_TYPE_TABLE: + opts->selTypes = 1; + opts->selTable = 1; + simple_string_list_append(&opts->tableNames, objname); + break; + case FILTER_OBJECT_TYPE_TRIGGER: + opts->selTypes = 1; + opts->selTrigger = 1; + simple_string_list_append(&opts->triggerNames, optarg); + break; + default: + pg_log_error("Unsupported include object"); + exit_nicely(1); + break; + } +} diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 8b9d9f4cad4..955bfcfdad2 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -779,6 +779,80 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term> + <listitem> + <para> + Specify a filename from which to read patterns for objects to include + or exclude from the dump. The patterns are interpreted according to the + same rules as the corresponding options: + <option>-t</option>/<option>--table</option> for tables, + <option>-n</option>/<option>--schema</option> for schemas, + <option>--include-foreign-data</option> for data on foreign servers and + <option>--exclude-table-data</option> for table data. + To read from <literal>STDIN</literal>, use <filename>-</filename> as the + filename. The <option>--filter</option> option can be specified in + conjunction with the above listed options for including or excluding + objects, and can also be specified more than once for multiple filter + files. + </para> + + <para> + The file lists one object pattern per row, with the following format: +<synopsis> +{ include | exclude } { table | schema | foreign_data | data } <replaceable class="parameter">PATTERN</replaceable> +</synopsis> + </para> + + <para> + The first keyword specifies whether the objects matched by the pattern + are to be included or excluded. The second keyword specifies the type + of object to be filtered using the pattern: + <itemizedlist> + <listitem> + <para> + <literal>table</literal>: tables, works like + <option>-t</option>/<option>--table</option> + </para> + </listitem> + <listitem> + <para> + <literal>schema</literal>: schemas, works like + <option>-n</option>/<option>--schema</option> + </para> + </listitem> + <listitem> + <para> + <literal>foreign_data</literal>: data on foreign servers, works like + <option>--include-foreign-data</option>. This keyword can only be + used with the <literal>include</literal> keyword. + </para> + </listitem> + <listitem> + <para> + <literal>data</literal>: table data, works like + <option>--exclude-table-data</option>. This keyword can only be + used with the <literal>exclude</literal> keyword. + </para> + </listitem> + </itemizedlist> + </para> + + <para> + Lines starting with <literal>#</literal> are considered comments and + ignored. Comments can be placed after filter as well. Blank lines + are also ignored. See <xref linkend="app-psql-patterns"/> for how to + perform quoting in patterns. + </para> + + <para> + Example files are listed below in the <xref linkend="pg-dump-examples"/> + section. + </para> + + </listitem> + </varlistentry> + <varlistentry> <term><option>--if-exists</option></term> <listitem> @@ -1119,6 +1193,7 @@ PostgreSQL documentation schema (<option>-n</option>/<option>--schema</option>) and table (<option>-t</option>/<option>--table</option>) qualifier match at least one extension/schema/table in the database to be dumped. + This also applies to filters used with <option>--filter</option>. Note that if none of the extension/schema/table qualifiers find matches, <application>pg_dump</application> will generate an error even without <option>--strict-names</option>. @@ -1528,6 +1603,19 @@ CREATE DATABASE foo WITH TEMPLATE template0; <screen> <prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput> +</screen></para> + + <para> + To dump all tables with names starting with mytable, except for table + <literal>mytable2</literal>, specify a filter file + <filename>filter.txt</filename> like: +<programlisting> +include table mytable* +exclude table mytable2 +</programlisting> + +<screen> +<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput> </screen></para> </refsect1> diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index e62d05e5ab5..7d8de30310b 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -122,6 +122,28 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term> + <listitem> + <para> + Specify a filename from which to read patterns for databases excluded + from dump. The patterns are interpreted according to the same rules + as <option>--exclude-database</option>. + To read from <literal>STDIN</literal>, use <filename>-</filename> as the + filename. The <option>--filter</option> option can be specified in + conjunction with the above listed options for excluding databases, + and can also be specified more than once for multiple filter files. + </para> + + <para> + The file lists one database pattern per row, with the following format: +<synopsis> +exclude database <replaceable class="parameter">PATTERN</replaceable> +</synopsis> + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>-g</option></term> <term><option>--globals-only</option></term> diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index 47bd7dbda06..ffeb564c520 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -188,6 +188,31 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term> + <listitem> + <para> + Specify a filename from which to read patterns for objects excluded + or included from restore. The patterns are interpretted according to the + same rules as <option>--schema</option>, <option>--exclude-schema</option>, + <option>--function</option>, <option>--index</option>, <option>--table</option> + or <option>--trigger</option>. + To read from <literal>STDIN</literal>, use <filename>-</filename> as the + filename. The <option>--filter</option> option can be specified in + conjunction with the above listed options for including or excluding + objects, and can also be specified more than once for multiple filter + files. + </para> + + <para> + The file lists one database pattern per row, with the following format: +<synopsis> +{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable> +</synopsis> + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>-F <replaceable class="parameter">format</replaceable></option></term> <term><option>--format=<replaceable class="parameter">format</replaceable></option></term> diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index ddb4f25eb12..4ae0c94319f 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -78,7 +78,9 @@ my $frontend_extraincludes = { my $frontend_extrasource = { 'psql' => ['src/bin/psql/psqlscanslash.l'], 'pgbench' => - [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ] + [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ], + 'pg_dump' => + ['src/bin/pg_dump/filterscan.l', 'src/bin/pg_dump/filterparse.y'] }; my @frontend_excludes = ( 'pgevent', 'pg_basebackup', 'pg_rewind', 'pg_dump', -- 2.37.3.542.gdd3f6c4cae
