po 8. 6. 2020 v 23:30 odesílatel Justin Pryzby <[email protected]>
napsal:
> On Mon, Jun 08, 2020 at 07:18:49PM +0200, Pavel Stehule wrote:
> > pá 29. 5. 2020 v 20:25 odesílatel Justin Pryzby <[email protected]>
> napsal:
> >
> > > On Fri, May 29, 2020 at 04:21:00PM +0200, Pavel Stehule wrote:
> > > > one my customer has to specify dumped tables name by name. After
> years and
> > > > increasing database size and table numbers he has problem with too
> short
> > > > command line. He need to read the list of tables from file (or from
> stdin).
> > >
> > > +1 - we would use this.
> > >
> > > We put a regex (actually a pg_dump pattern) of tables to skip
> (timeseries
> > > partitions which are older than a few days and which are also dumped
> once not
> > > expected to change, and typically not redumped). We're nowhere near
> the
> > > execve() limit, but it'd be nice if the command was primarily a list
> of options
> > > and not a long regex.
> > >
> > > Please also support reading from file for --exclude-table=pattern.
> > >
> > > I'm drawing a parallel between this and rsync --include/--exclude and
> > > --filter.
> > >
> > > We'd be implementing a new --filter, which might have similar syntax
> to rsync
> > > (which I always forget).
> >
> > I implemented support for all "repeated" pg_dump options.
> >
> > I invite any help with doc. There is just very raw text
> >
> > + Do not dump data of tables spefified in file.
>
> *specified
>
>
I am sending updated version - now with own implementation GNU (not POSIX)
function getline
I still wonder if a better syntax would use a unified --filter option, whose
> argument would allow including/excluding any type of object:
>
> +[tn] include (t)table/(n)namespace/...
> -[tn] exclude (t)table/(n)namespace/...
>
> In the past, I looked for a way to exclude extended stats objects, and
> ended up
> using a separate schema. An "extensible" syntax might be better (although
> reading a file of just patterns has the advantage that the function can
> just be
> called once for each option for each type of object).
>
I tried to implement simple format "[+-][tndf] objectname"
please, check attached patch
Regards
Pavel
>
> --
> Justin
>
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2f0807e912..332f0c312f 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -813,6 +813,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Read filters from file. Format "(+|-)(tnfd) objectname:
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--load-via-partition-root</option></term>
<listitem>
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index dfe43968b8..acc936ac7b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -290,6 +290,8 @@ static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+static size_t pg_getline(char **lineptr, size_t *n, FILE *fp);
+static void invalid_filter_format(char *filename, char *line, int lineno);
int
@@ -364,6 +366,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 12},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -603,6 +606,116 @@ main(int argc, char **argv)
optarg);
break;
+ case 12: /* filter implementation */
+ {
+ FILE *f;
+ char *line;
+ ssize_t chars;
+ size_t line_size = 1024;
+ int lineno = 0;
+
+ /* use "-" as symbol for stdin */
+ if (strcmp(optarg, "-") != 0)
+ {
+ f = fopen(optarg, "r");
+ if (!f)
+ fatal("could not open the input file \"%s\": %s",
+ optarg,
+ strerror(errno));
+ }
+ else
+ f = stdin;
+
+ line = pg_malloc(line_size);
+
+ while ((chars = pg_getline(&line, &line_size, f)) != -1)
+ {
+ lineno += 1;
+
+ if (line[chars - 1] == '\n')
+ line[chars - 1] = '\0';
+
+ /* ignore empty rows */
+ if (*line != '\0')
+ {
+ bool include_filter = false;
+ bool exclude_filter = false;
+ char objecttype;
+ char *objectname;
+
+ if (chars < 4)
+ invalid_filter_format(optarg, line, lineno);
+
+ if (line[0] == '+')
+ include_filter = true;
+ else if (line[0] == '-')
+ exclude_filter = true;
+ else
+ invalid_filter_format(optarg, line, lineno);
+
+ objecttype = line[1];
+
+ if (objecttype != 't' &&
+ objecttype != 'n' &&
+ objecttype != 'd' &&
+ objecttype != 'f')
+ invalid_filter_format(optarg, line, lineno);
+
+ objectname = &line[3];
+
+ /* skip initial spaces */
+ while (*objectname == ' ')
+ objectname++;
+
+ if (objecttype == 't')
+ {
+ if (include_filter)
+ {
+ simple_string_list_append(&table_include_patterns, objectname);
+ dopt.include_everything = false;
+ }
+ else if (exclude_filter)
+ simple_string_list_append(&table_exclude_patterns, objectname);
+ }
+ else if (objecttype == 'n')
+ {
+ if (include_filter)
+ {
+ simple_string_list_append(&schema_include_patterns, objectname);
+ dopt.include_everything = false;
+ }
+ else if (exclude_filter)
+ simple_string_list_append(&schema_exclude_patterns, objectname);
+ }
+ else if (objecttype == 'd')
+ {
+ if (include_filter)
+ invalid_filter_format(optarg, line, lineno);
+ else if (exclude_filter)
+ simple_string_list_append(&tabledata_exclude_patterns, objectname);
+ }
+ else if (objecttype == 'f')
+ {
+ if (include_filter)
+ simple_string_list_append(&foreign_servers_include_patterns, objectname);
+ else if (exclude_filter)
+ invalid_filter_format(optarg, line, lineno);
+ }
+ }
+ }
+
+ if (ferror(f))
+ fatal("could not read from file \"%s\": %s",
+ f == stdin ? "stdin" : optarg,
+ strerror(errno));
+
+ if (f != stdin)
+ fclose(f);
+
+ pg_free(line);
+ }
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit_nicely(1);
@@ -1022,6 +1135,7 @@ help(const char *progname)
" access to)\n"));
printf(_(" --exclude-table-data=PATTERN do NOT dump data for the specified table(s)\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME read object names from file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18647,3 +18761,61 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse reloptions array");
}
+
+/*
+ * getline is originaly GNU function, and should not be everywhere still.
+ * Use own reduced implementation.
+ */
+static size_t
+pg_getline(char **lineptr, size_t *n, FILE *fp)
+{
+ size_t total_chars = 0;
+
+ while (!feof(fp) && !ferror(fp))
+ {
+ char *str;
+ size_t chars;
+
+ str = fgets(*lineptr + total_chars,
+ *n - total_chars,
+ fp);
+
+ if (ferror(fp))
+ return -1;
+
+ if (str)
+ {
+ chars = strlen(str);
+ total_chars += chars;
+
+ if (chars > 0 && str[chars - 1] == '\n')
+ return total_chars;
+
+ /* check, if there is good enough space for next content */
+ if (*n - total_chars < 2)
+ {
+ *n += 1024;
+ *lineptr = pg_realloc(*lineptr, *n);
+ }
+ }
+ else
+ break;
+ }
+
+ if (ferror(fp))
+ return -1;
+
+ return total_chars > 0 ? total_chars : -1;
+}
+
+/*
+ * Print error message and exit.
+ */
+static void
+invalid_filter_format(char *filename, char *line, int lineno)
+{
+ pg_log_error("invalid format of filter file \"%s\"",
+ *filename == '-' ? "stdin" : filename);
+ fprintf(stderr, "%d: %s\n", lineno, line);
+ exit_nicely(1);
+}
