From 7666c6d2321b696c5b4a7dd4c4c80f7b12b22813 Mon Sep 17 00:00:00 2001 From: Srinath Reddy Sadipiralla Date: Tue, 28 Jan 2025 11:41:14 +0530 Subject: [PATCH 1/1] Refactor command_fails_like in pg_dump/t/001_basic.pl to pass make check-world. --- doc/src/sgml/ref/pg_dumpall.sgml | 78 ++- doc/src/sgml/ref/pg_restore.sgml | 29 + src/bin/pg_dump/Makefile | 8 +- src/bin/pg_dump/common_dumpall_restore.c | 314 ++++++++++ src/bin/pg_dump/common_dumpall_restore.h | 26 + src/bin/pg_dump/meson.build | 2 + src/bin/pg_dump/pg_backup.h | 4 +- src/bin/pg_dump/pg_backup_archiver.c | 15 +- src/bin/pg_dump/pg_backup_tar.c | 2 +- src/bin/pg_dump/pg_backup_utils.c | 3 +- src/bin/pg_dump/pg_dump.c | 2 +- src/bin/pg_dump/pg_dumpall.c | 516 +++++++---------- src/bin/pg_dump/pg_restore.c | 701 ++++++++++++++++++++++- src/bin/pg_dump/t/001_basic.pl | 11 + src/tools/pgindent/typedefs.list | 2 + 15 files changed, 1373 insertions(+), 340 deletions(-) create mode 100644 src/bin/pg_dump/common_dumpall_restore.c create mode 100644 src/bin/pg_dump/common_dumpall_restore.h mode change 100644 => 100755 src/bin/pg_dump/t/001_basic.pl diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 014f279258..8ca49a6597 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -121,7 +121,83 @@ PostgreSQL documentation Send output to the specified file. If this is omitted, the standard output is used. - + Note: This option can be omitted only when is plain + + + + + + + + + + Specify format of dump files. If we want to dump all the databases, + then pass this as non-plain so that dump of all databases can be taken + in separate subdirectory in archive format. + by default, this is plain format. + + If non-plain mode is passed, then global.dat (global sql commands) and + map.dat(dboid and dbnames list of all the databases) files will be created. + Apart from these files, one subdirectory with databases name will be created. + Under this databases subdirectory, there will be files with dboid name for each + database and if is directory, then toc.dat and other + dump files will be under dboid subdirectory. + + + + d + directory + + + Output a directory-format archive suitable for input into pg_restore. Under dboid + subdirectory, this will create a directory with one file for each table and large + object being dumped, plus a so-called Table of Contents file describing the dumped + objects in a machine-readable format that pg_restore can read. A directory format + archive can be manipulated with standard Unix tools; for example, files in an + uncompressed archive can be compressed with the gzip, lz4, or zstd tools. This + format is compressed by default using gzip and also supports parallel dumps. + + + + + + p + plain + + + Output a plain-text SQL script file (the default). + + + + + + c + custom + + + Output a custom-format archive suitable for input into pg_restore. Together with the + directory output format, this is the most flexible output format in that it allows manual + selection and reordering of archived items during restore. This format is also + compressed by default. + + + + + + t + tar + + + Output a tar-format archive suitable for input into pg_restore. The tar format is + compatible with the directory format: extracting a tar-format archive produces a valid + directory-format archive. However, the tar format does not support compression. Also, + when using tar format the relative order of table data items cannot be changed during restore. + + + + + + diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index b8b27e1719..ba2913b335 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -166,6 +166,25 @@ PostgreSQL documentation + + + + + Do not restore databases whose name matches + pattern. + Multiple patterns can be excluded by writing multiple + switches. The + pattern parameter is + interpreted as a pattern according to the same rules used by + psql's \d + commands (see ), + so multiple databases can also be excluded by writing wildcard + characters in the pattern. When using wildcards, be careful to + quote the pattern if needed to prevent shell wildcard expansion. + + + + @@ -315,6 +334,16 @@ PostgreSQL documentation + + + + + + Restore only global objects (roles and tablespaces), no databases. + + + + diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 233ad15ca7..a4e557d62c 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -47,11 +47,11 @@ all: pg_dump pg_restore pg_dumpall pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) pg_dump.o common.o pg_dump_sort.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils - $(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_restore: pg_restore.o common_dumpall_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) pg_restore.o common_dumpall_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils - $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_dumpall: pg_dumpall.o dumputils.o filter.o common_dumpall_restore.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o common_dumpall_restore.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) diff --git a/src/bin/pg_dump/common_dumpall_restore.c b/src/bin/pg_dump/common_dumpall_restore.c new file mode 100644 index 0000000000..ace5077085 --- /dev/null +++ b/src/bin/pg_dump/common_dumpall_restore.c @@ -0,0 +1,314 @@ +/*------------------------------------------------------------------------- + * + * common_dumpall_restore.c + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * this is a common file for pg_dumpall and pg_restore. + * src/bin/pg_dump/common_dumpall_restore.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "common/connect.h" +#include "common/logging.h" +#include "common/string.h" +#include "common_dumpall_restore.h" +#include "dumputils.h" +#include "fe_utils/string_utils.h" + +static char *constructConnStr(const char **keywords, const char **values); +#define exit_nicely(code) exit(code) + +/* + * Make a database connection with the given parameters. An + * interactive password prompt is automatically issued if required. + * + * If fail_on_error is false, we return NULL without printing any message + * on failure, but preserve any prompted password for the next try. + * + * On success, the global variable 'connstr' is set to a connection string + * containing the options used. + */ +PGconn * +connectDatabase(const char *dbname, const char *connection_string, + const char *pghost, const char *pgport, const char *pguser, + trivalue prompt_password, bool fail_on_error, const char *progname, + const char **connstr, int *server_version) +{ + PGconn *conn; + bool new_pass; + const char *remoteversion_str; + int my_version; + const char **keywords = NULL; + const char **values = NULL; + PQconninfoOption *conn_opts = NULL; + static char *password = NULL; + int server_version_temp; + + if (prompt_password == TRI_YES && !password) + password = simple_prompt("Password: ", false); + + /* + * Start the connection. Loop until we have a password if requested by + * backend. + */ + do + { + int argcount = 6; + PQconninfoOption *conn_opt; + char *err_msg = NULL; + int i = 0; + + free(keywords); + free(values); + PQconninfoFree(conn_opts); + + /* + * Merge the connection info inputs given in form of connection string + * and other options. Explicitly discard any dbname value in the + * connection string; otherwise, PQconnectdbParams() would interpret + * that value as being itself a connection string. + */ + if (connection_string) + { + conn_opts = PQconninfoParse(connection_string, &err_msg); + if (conn_opts == NULL) + pg_fatal("%s", err_msg); + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (conn_opt->val != NULL && conn_opt->val[0] != '\0' && + strcmp(conn_opt->keyword, "dbname") != 0) + argcount++; + } + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (conn_opt->val != NULL && conn_opt->val[0] != '\0' && + strcmp(conn_opt->keyword, "dbname") != 0) + { + keywords[i] = conn_opt->keyword; + values[i] = conn_opt->val; + i++; + } + } + } + else + { + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + } + + if (pghost) + { + keywords[i] = "host"; + values[i] = pghost; + i++; + } + if (pgport) + { + keywords[i] = "port"; + values[i] = pgport; + i++; + } + if (pguser) + { + keywords[i] = "user"; + values[i] = pguser; + i++; + } + if (password) + { + keywords[i] = "password"; + values[i] = password; + i++; + } + if (dbname) + { + keywords[i] = "dbname"; + values[i] = dbname; + i++; + } + keywords[i] = "fallback_application_name"; + values[i] = progname; + i++; + + new_pass = false; + conn = PQconnectdbParams(keywords, values, true); + + if (!conn) + pg_fatal("could not connect to database \"%s\"", dbname); + + if (PQstatus(conn) == CONNECTION_BAD && + PQconnectionNeedsPassword(conn) && + !password && + prompt_password != TRI_NO) + { + PQfinish(conn); + password = simple_prompt("Password: ", false); + new_pass = true; + } + } while (new_pass); + + /* check to see that the backend connection was successfully made */ + if (PQstatus(conn) == CONNECTION_BAD) + { + if (fail_on_error) + pg_fatal("%s", PQerrorMessage(conn)); + else + { + PQfinish(conn); + + free(keywords); + free(values); + PQconninfoFree(conn_opts); + + return NULL; + } + } + + /* + * Ok, connected successfully. Remember the options used, in the form of a + * connection string. + */ + if (connstr) + *connstr = constructConnStr(keywords, values); + + free(keywords); + free(values); + PQconninfoFree(conn_opts); + + /* Check version */ + remoteversion_str = PQparameterStatus(conn, "server_version"); + if (!remoteversion_str) + pg_fatal("could not get server version"); + server_version_temp = PQserverVersion(conn); + if (server_version_temp == 0) + pg_fatal("could not parse server version \"%s\"", + remoteversion_str); + + /* If needed, then copy server version to outer function. */ + if (server_version) + *server_version = server_version_temp; + + my_version = PG_VERSION_NUM; + + /* + * We allow the server to be back to 9.2, and up to any minor release of + * our own major version. (See also version check in pg_dump.c.) + */ + if (my_version != server_version_temp + && (server_version_temp < 90200 || + (server_version_temp / 100) > (my_version / 100))) + { + pg_log_error("aborting because of server version mismatch"); + pg_log_error_detail("server version: %s; %s version: %s", + remoteversion_str, progname, PG_VERSION); + exit_nicely(1); + } + + PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL)); + + return conn; +} + +/* ---------- + * Construct a connection string from the given keyword/value pairs. It is + * used to pass the connection options to the pg_dump subprocess. + * + * The following parameters are excluded: + * dbname - varies in each pg_dump invocation + * password - it's not secure to pass a password on the command line + * fallback_application_name - we'll let pg_dump set it + * ---------- + */ +static char * +constructConnStr(const char **keywords, const char **values) +{ + PQExpBuffer buf = createPQExpBuffer(); + char *connstr; + int i; + bool firstkeyword = true; + + /* Construct a new connection string in key='value' format. */ + for (i = 0; keywords[i] != NULL; i++) + { + if (strcmp(keywords[i], "dbname") == 0 || + strcmp(keywords[i], "password") == 0 || + strcmp(keywords[i], "fallback_application_name") == 0) + continue; + + if (!firstkeyword) + appendPQExpBufferChar(buf, ' '); + firstkeyword = false; + appendPQExpBuffer(buf, "%s=", keywords[i]); + appendConnStrVal(buf, values[i]); + } + + connstr = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + return connstr; +} + +/* + * Run a query, return the results, exit program on failure. + */ +PGresult * +executeQuery(PGconn *conn, const char *query) +{ + PGresult *res; + + pg_log_info("executing %s", query); + + res = PQexec(conn, query); + if (!res || + PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("query failed: %s", PQerrorMessage(conn)); + pg_log_error_detail("Query was: %s", query); + PQfinish(conn); + exit_nicely(1); + } + + return res; +} + +/* + * parseDumpFormat + * + * This will validate dump formats. + */ +ArchiveFormat +parseDumpFormat(const char *format) +{ + ArchiveFormat archDumpFormat; + + if (pg_strcasecmp(format, "c") == 0) + archDumpFormat = archCustom; + else if (pg_strcasecmp(format, "custom") == 0) + archDumpFormat = archCustom; + else if (pg_strcasecmp(format, "d") == 0) + archDumpFormat = archDirectory; + else if (pg_strcasecmp(format, "directory") == 0) + archDumpFormat = archDirectory; + else if (pg_strcasecmp(format, "p") == 0) + archDumpFormat = archNull; + else if (pg_strcasecmp(format, "plain") == 0) + archDumpFormat = archNull; + else if (pg_strcasecmp(format, "t") == 0) + archDumpFormat = archTar; + else if (pg_strcasecmp(format, "tar") == 0) + archDumpFormat = archTar; + else + pg_fatal("unrecognized archive format \"%s\"; please specify \"c\", \"d\", or \"t\"", + format); + + return archDumpFormat; +} diff --git a/src/bin/pg_dump/common_dumpall_restore.h b/src/bin/pg_dump/common_dumpall_restore.h new file mode 100644 index 0000000000..a27c3e9fb8 --- /dev/null +++ b/src/bin/pg_dump/common_dumpall_restore.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * common_dumpall_restore.h + * Common header file for pg_dumpall and pg_restore + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/bin/pg_dump/common_dumpall_restore.h + * + *------------------------------------------------------------------------- + */ +#ifndef COMMON_DUMPALL_RESTORE_H +#define COMMON_DUMPALL_RESTORE_H + +#include "pg_backup.h" + +extern PGconn *connectDatabase(const char *dbname, + const char *connection_string, const char *pghost, + const char *pgport, const char *pguser, + trivalue prompt_password, bool fail_on_error, + const char *progname, const char **connstr, int *server_version); +extern PGresult *executeQuery(PGconn *conn, const char *query); +extern ArchiveFormat parseDumpFormat(const char *format); +#endif /* COMMON_DUMPALL_RESTORE_H */ diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build index 603ba6cfbf..ddecac5cf0 100644 --- a/src/bin/pg_dump/meson.build +++ b/src/bin/pg_dump/meson.build @@ -49,6 +49,7 @@ bin_targets += pg_dump pg_dumpall_sources = files( 'pg_dumpall.c', + 'common_dumpall_restore.c', ) if host_system == 'windows' @@ -68,6 +69,7 @@ bin_targets += pg_dumpall pg_restore_sources = files( 'pg_restore.c', + 'common_dumpall_restore.c', ) if host_system == 'windows' diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index f0f19bb0b2..65000e5a08 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -306,7 +306,7 @@ extern void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ro extern void ProcessArchiveRestoreOptions(Archive *AHX); -extern void RestoreArchive(Archive *AHX); +extern void RestoreArchive(Archive *AHX, bool append_data); /* Open an existing archive */ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); @@ -319,7 +319,7 @@ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, DataDirSyncMethod sync_method); /* The --list option */ -extern void PrintTOCSummary(Archive *AHX); +extern void PrintTOCSummary(Archive *AHX, bool append_data); extern RestoreOptions *NewRestoreOptions(void); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 707a3fc844..7153d4a40b 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -82,7 +82,7 @@ static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); static void SetOutput(ArchiveHandle *AH, const char *filename, - const pg_compress_specification compression_spec); + const pg_compress_specification compression_spec, bool append_data); static CompressFileHandle *SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput); @@ -333,7 +333,7 @@ ProcessArchiveRestoreOptions(Archive *AHX) /* Public */ void -RestoreArchive(Archive *AHX) +RestoreArchive(Archive *AHX, bool append_data) { ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; @@ -450,7 +450,7 @@ RestoreArchive(Archive *AHX) */ sav = SaveOutput(AH); if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE) - SetOutput(AH, ropt->filename, ropt->compression_spec); + SetOutput(AH, ropt->filename, ropt->compression_spec, append_data); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -1263,7 +1263,7 @@ ArchiveEntry(Archive *AHX, CatalogId catalogId, DumpId dumpId, /* Public */ void -PrintTOCSummary(Archive *AHX) +PrintTOCSummary(Archive *AHX, bool append_data) { ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; @@ -1279,7 +1279,7 @@ PrintTOCSummary(Archive *AHX) sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, out_compression_spec); + SetOutput(AH, ropt->filename, out_compression_spec, append_data); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1658,7 +1658,8 @@ archprintf(Archive *AH, const char *fmt,...) static void SetOutput(ArchiveHandle *AH, const char *filename, - const pg_compress_specification compression_spec) + const pg_compress_specification compression_spec, + bool append_data) { CompressFileHandle *CFH; const char *mode; @@ -1678,7 +1679,7 @@ SetOutput(ArchiveHandle *AH, const char *filename, else fn = fileno(stdout); - if (AH->mode == archModeAppend) + if (append_data || AH->mode == archModeAppend) mode = PG_BINARY_A; else mode = PG_BINARY_W; diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index b5ba3b46dd..d94d0de2a5 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -826,7 +826,7 @@ _CloseArchive(ArchiveHandle *AH) savVerbose = AH->public.verbose; AH->public.verbose = 0; - RestoreArchive((Archive *) AH); + RestoreArchive((Archive *) AH, false); SetArchiveOptions((Archive *) AH, savDopt, savRopt); diff --git a/src/bin/pg_dump/pg_backup_utils.c b/src/bin/pg_dump/pg_backup_utils.c index 79aec5f515..f70ea9233f 100644 --- a/src/bin/pg_dump/pg_backup_utils.c +++ b/src/bin/pg_dump/pg_backup_utils.c @@ -21,7 +21,8 @@ /* Globals exported by this file */ const char *progname = NULL; -#define MAX_ON_EXIT_NICELY 20 +/* TODO: increasing this to keep 100 db restoring by single pg_restore command. */ +#define MAX_ON_EXIT_NICELY 100 static struct { diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index af857f00c7..2de2621c8f 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1148,7 +1148,7 @@ main(int argc, char **argv) * right now. */ if (plainText) - RestoreArchive(fout); + RestoreArchive(fout, false); CloseArchive(fout); diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 396f79781c..5915b1b051 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -15,6 +15,7 @@ #include "postgres_fe.h" +#include #include #include @@ -24,14 +25,17 @@ #include "common/hashfn_unstable.h" #include "common/logging.h" #include "common/string.h" +#include "common_dumpall_restore.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "filter.h" #include "getopt_long.h" #include "pg_backup.h" +#include "pg_backup_archiver.h" /* version string we expect back from pg_dump */ #define PGDUMP_VERSIONSTR "pg_dump (PostgreSQL) " PG_VERSION "\n" +#define exit_nicely(code) exit(code) typedef struct { @@ -64,28 +68,24 @@ static void dropTablespaces(PGconn *conn); static void dumpTablespaces(PGconn *conn); static void dropDBs(PGconn *conn); static void dumpUserConfig(PGconn *conn, const char *username); -static void dumpDatabases(PGconn *conn); +static void dumpDatabases(PGconn *conn, ArchiveFormat archDumpFormat); static void dumpTimestamp(const char *msg); -static int runPgDump(const char *dbname, const char *create_opts); +static int runPgDump(const char *dbname, const char *create_opts, + char *dbfile, ArchiveFormat archDumpFormat); static void buildShSecLabels(PGconn *conn, const char *catalog_name, Oid objectId, const char *objtype, const char *objname, PQExpBuffer buffer); -static PGconn *connectDatabase(const char *dbname, - const char *connection_string, const char *pghost, - const char *pgport, const char *pguser, - trivalue prompt_password, bool fail_on_error); -static char *constructConnStr(const char **keywords, const char **values); -static PGresult *executeQuery(PGconn *conn, const char *query); static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); static void read_dumpall_filters(const char *filename, SimpleStringList *pattern); +static void create_or_open_dir(const char *dirname); static char pg_dump_bin[MAXPGPATH]; static const char *progname; static PQExpBuffer pgdumpopts; -static char *connstr = ""; +static const char *connstr = ""; static bool output_clean = false; static bool skip_acls = false; static bool verbose = false; @@ -107,7 +107,7 @@ static int no_subscriptions = 0; static int no_toast_compression = 0; static int no_unlogged_table_data = 0; static int no_role_passwords = 0; -static int server_version; +static int server_version; static int load_via_partition_root = 0; static int on_conflict_do_nothing = 0; @@ -121,8 +121,6 @@ static char *filename = NULL; static SimpleStringList database_exclude_patterns = {NULL, NULL}; static SimpleStringList database_exclude_names = {NULL, NULL}; -#define exit_nicely(code) exit(code) - int main(int argc, char *argv[]) { @@ -147,6 +145,7 @@ main(int argc, char *argv[]) {"password", no_argument, NULL, 'W'}, {"no-privileges", no_argument, NULL, 'x'}, {"no-acl", no_argument, NULL, 'x'}, + {"format", required_argument, NULL, 'F'}, /* * the following options don't have an equivalent short option letter @@ -188,6 +187,8 @@ main(int argc, char *argv[]) char *pgdb = NULL; char *use_role = NULL; const char *dumpencoding = NULL; + ArchiveFormat archDumpFormat = archNull; + const char *formatName = "p"; trivalue prompt_password = TRI_DEFAULT; bool data_only = false; bool globals_only = false; @@ -237,7 +238,7 @@ main(int argc, char *argv[]) pgdumpopts = createPQExpBuffer(); - while ((c = getopt_long(argc, argv, "acd:E:f:gh:l:Op:rsS:tU:vwWx", long_options, &optindex)) != -1) + while ((c = getopt_long(argc, argv, "acd:E:f:F:gh:l:Op:rsS:tU:vwWx", long_options, &optindex)) != -1) { switch (c) { @@ -265,7 +266,9 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " -f "); appendShellString(pgdumpopts, filename); break; - + case 'F': + formatName = pg_strdup(optarg); + break; case 'g': globals_only = true; break; @@ -414,6 +417,21 @@ main(int argc, char *argv[]) exit_nicely(1); } + /* Get format for dump. */ + archDumpFormat = parseDumpFormat(formatName); + + /* + * If non-plain format is specified then we must provide the + * file name to create one main directory. + */ + if (archDumpFormat != archNull && + (!filename || strcmp(filename, "") == 0)) + { + pg_log_error("options -F/--format=d|c|t requires option -f/--file with non-empty string"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit_nicely(1); + } + /* * If password values are not required in the dump, switch to using * pg_roles which is equally useful, just more likely to have unrestricted @@ -460,6 +478,33 @@ main(int argc, char *argv[]) if (on_conflict_do_nothing) appendPQExpBufferStr(pgdumpopts, " --on-conflict-do-nothing"); + /* + * Open the output file if required, otherwise use stdout. If required, + * then create new directory and global.dat file. + */ + if (archDumpFormat != archNull) + { + char toc_path[MAXPGPATH]; + + /* Create new directory or accept the empty existing directory. */ + create_or_open_dir(filename); + + snprintf(toc_path, MAXPGPATH, "%s/global.dat", filename); + + OPF = fopen(toc_path, PG_BINARY_W); + if (!OPF) + pg_fatal("could not open global.dat file: %s", strerror(errno)); + } + else if (filename) + { + OPF = fopen(filename, PG_BINARY_W); + if (!OPF) + pg_fatal("could not open output file \"%s\": %m", + filename); + } + else + OPF = stdout; + /* * If there was a database specified on the command line, use that, * otherwise try to connect to database "postgres", and failing that @@ -468,7 +513,8 @@ main(int argc, char *argv[]) if (pgdb) { conn = connectDatabase(pgdb, connstr, pghost, pgport, pguser, - prompt_password, false); + prompt_password, false, + progname, &connstr, &server_version); if (!conn) pg_fatal("could not connect to database \"%s\"", pgdb); @@ -476,10 +522,12 @@ main(int argc, char *argv[]) else { conn = connectDatabase("postgres", connstr, pghost, pgport, pguser, - prompt_password, false); + prompt_password, false, + progname, &connstr, &server_version); if (!conn) conn = connectDatabase("template1", connstr, pghost, pgport, pguser, - prompt_password, true); + prompt_password, true, + progname, &connstr, &server_version); if (!conn) { @@ -496,19 +544,6 @@ main(int argc, char *argv[]) expand_dbname_patterns(conn, &database_exclude_patterns, &database_exclude_names); - /* - * Open the output file if required, otherwise use stdout - */ - if (filename) - { - OPF = fopen(filename, PG_BINARY_W); - if (!OPF) - pg_fatal("could not open output file \"%s\": %m", - filename); - } - else - OPF = stdout; - /* * Set the client encoding if requested. */ @@ -607,7 +642,7 @@ main(int argc, char *argv[]) } if (!globals_only && !roles_only && !tablespaces_only) - dumpDatabases(conn); + dumpDatabases(conn, archDumpFormat); PQfinish(conn); @@ -620,7 +655,7 @@ main(int argc, char *argv[]) fclose(OPF); /* sync the resulting file, errors are not fatal */ - if (dosync) + if (dosync && (archDumpFormat == archNull)) (void) fsync_fname(filename, false); } @@ -637,6 +672,8 @@ help(void) printf(_("\nGeneral options:\n")); printf(_(" -f, --file=FILENAME output file name\n")); + printf(_(" -F, --format=c|d|t|p output file format (custom, directory, tar,\n" + " plain text (default))\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); @@ -1487,10 +1524,13 @@ expand_dbname_patterns(PGconn *conn, * Dump contents of databases. */ static void -dumpDatabases(PGconn *conn) +dumpDatabases(PGconn *conn, ArchiveFormat archDumpFormat) { PGresult *res; int i; + char db_subdir[MAXPGPATH]; + char dbfilepath[MAXPGPATH]; + FILE *map_file = NULL; /* * Skip databases marked not datallowconn, since we'd be unable to connect @@ -1504,7 +1544,7 @@ dumpDatabases(PGconn *conn) * doesn't have some failure mode with --clean. */ res = executeQuery(conn, - "SELECT datname " + "SELECT datname, oid " "FROM pg_database d " "WHERE datallowconn AND datconnlimit != -2 " "ORDER BY (datname <> 'template1'), datname"); @@ -1512,9 +1552,33 @@ dumpDatabases(PGconn *conn) if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Databases\n--\n\n"); + /* + * If directory/tar/custom format is specified then create a subdirectory + * under the main directory and each database dump file subdirectory will + * be created under the subdirectory in archive mode as per single db pg_dump. + */ + if (archDumpFormat != archNull) + { + char map_file_path[MAXPGPATH]; + + snprintf(db_subdir, MAXPGPATH, "%s/databases", filename); + + /* Create a subdirectory with 'databases' name under main directory. */ + if (mkdir(db_subdir, 0755) != 0) + pg_log_error("could not create subdirectory \"%s\": %m", db_subdir); + + snprintf(map_file_path, MAXPGPATH, "%s/map.dat", filename); + + /* Create a map file (to store dboid and dbname) */ + map_file = fopen(map_file_path, PG_BINARY_W); + if (!map_file) + pg_fatal("could not open map file: %s", strerror(errno)); + } + for (i = 0; i < PQntuples(res); i++) { char *dbname = PQgetvalue(res, i, 0); + char *oid = PQgetvalue(res, i, 1); const char *create_opts; int ret; @@ -1529,6 +1593,18 @@ dumpDatabases(PGconn *conn) continue; } + /* + * If this is non-plain dump format, then append dboid and dbname to + * the map.dat file. + */ + if (archDumpFormat != archNull) + { + snprintf(dbfilepath, MAXPGPATH, "\"%s\"/\"%s\"", db_subdir, oid); + + /* Put one line entry for dboid and dbname in map file. */ + fprintf(map_file, "%s %s\n", oid, pg_strdup(dbname)); + } + pg_log_info("dumping database \"%s\"", dbname); fprintf(OPF, "--\n-- Database \"%s\" dump\n--\n\n", dbname); @@ -1547,9 +1623,17 @@ dumpDatabases(PGconn *conn) create_opts = "--clean --create"; else { - create_opts = ""; /* Since pg_dump won't emit a \connect command, we must */ - fprintf(OPF, "\\connect %s\n\n", dbname); + if (archDumpFormat == archNull) + { + create_opts = ""; + fprintf(OPF, "\\connect %s\n\n", dbname); + } + else + { + /* Dumping all databases so add --create option. */ + create_opts = "--create"; + } } } else @@ -1558,19 +1642,30 @@ dumpDatabases(PGconn *conn) if (filename) fclose(OPF); - ret = runPgDump(dbname, create_opts); + ret = runPgDump(dbname, create_opts, dbfilepath, archDumpFormat); if (ret != 0) pg_fatal("pg_dump failed on database \"%s\", exiting", dbname); if (filename) { - OPF = fopen(filename, PG_BINARY_A); + char toc_path[MAXPGPATH]; + + if (archDumpFormat != archNull) + snprintf(toc_path, MAXPGPATH, "%s/global.dat", filename); + else + snprintf(toc_path, MAXPGPATH, "%s", filename); + + OPF = fopen(toc_path, PG_BINARY_A); if (!OPF) pg_fatal("could not re-open the output file \"%s\": %m", - filename); + toc_path); } } + /* Close map file */ + if (archDumpFormat != archNull) + fclose(map_file); + PQclear(res); } @@ -1580,7 +1675,8 @@ dumpDatabases(PGconn *conn) * Run pg_dump on dbname, with specified options. */ static int -runPgDump(const char *dbname, const char *create_opts) +runPgDump(const char *dbname, const char *create_opts, char *dbfile, + ArchiveFormat archDumpFormat) { PQExpBufferData connstrbuf; PQExpBufferData cmd; @@ -1589,17 +1685,36 @@ runPgDump(const char *dbname, const char *create_opts) initPQExpBuffer(&connstrbuf); initPQExpBuffer(&cmd); - printfPQExpBuffer(&cmd, "\"%s\" %s %s", pg_dump_bin, - pgdumpopts->data, create_opts); - /* - * If we have a filename, use the undocumented plain-append pg_dump - * format. + * If this is non-plain format dump, then append file name and dump + * format to the pg_dump command to get archive dump. */ - if (filename) - appendPQExpBufferStr(&cmd, " -Fa "); + if (archDumpFormat != archNull) + { + printfPQExpBuffer(&cmd, "\"%s\" -f %s %s", pg_dump_bin, + dbfile, create_opts); + + if (archDumpFormat == archDirectory) + appendPQExpBufferStr(&cmd, " --format=directory "); + else if (archDumpFormat == archCustom) + appendPQExpBufferStr(&cmd, " --format=custom "); + else if (archDumpFormat == archTar) + appendPQExpBufferStr(&cmd, " --format=tar "); + } else - appendPQExpBufferStr(&cmd, " -Fp "); + { + printfPQExpBuffer(&cmd, "\"%s\" %s %s", pg_dump_bin, + pgdumpopts->data, create_opts); + + /* + * If we have a filename, use the undocumented plain-append pg_dump + * format. + */ + if (filename) + appendPQExpBufferStr(&cmd, " -Fa "); + else + appendPQExpBufferStr(&cmd, " -Fp "); + } /* * Append the database name to the already-constructed stem of connection @@ -1649,256 +1764,6 @@ buildShSecLabels(PGconn *conn, const char *catalog_name, Oid objectId, destroyPQExpBuffer(sql); } -/* - * Make a database connection with the given parameters. An - * interactive password prompt is automatically issued if required. - * - * If fail_on_error is false, we return NULL without printing any message - * on failure, but preserve any prompted password for the next try. - * - * On success, the global variable 'connstr' is set to a connection string - * containing the options used. - */ -static PGconn * -connectDatabase(const char *dbname, const char *connection_string, - const char *pghost, const char *pgport, const char *pguser, - trivalue prompt_password, bool fail_on_error) -{ - PGconn *conn; - bool new_pass; - const char *remoteversion_str; - int my_version; - const char **keywords = NULL; - const char **values = NULL; - PQconninfoOption *conn_opts = NULL; - static char *password = NULL; - - if (prompt_password == TRI_YES && !password) - password = simple_prompt("Password: ", false); - - /* - * Start the connection. Loop until we have a password if requested by - * backend. - */ - do - { - int argcount = 6; - PQconninfoOption *conn_opt; - char *err_msg = NULL; - int i = 0; - - free(keywords); - free(values); - PQconninfoFree(conn_opts); - - /* - * Merge the connection info inputs given in form of connection string - * and other options. Explicitly discard any dbname value in the - * connection string; otherwise, PQconnectdbParams() would interpret - * that value as being itself a connection string. - */ - if (connection_string) - { - conn_opts = PQconninfoParse(connection_string, &err_msg); - if (conn_opts == NULL) - pg_fatal("%s", err_msg); - - for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) - { - if (conn_opt->val != NULL && conn_opt->val[0] != '\0' && - strcmp(conn_opt->keyword, "dbname") != 0) - argcount++; - } - - keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); - values = pg_malloc0((argcount + 1) * sizeof(*values)); - - for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) - { - if (conn_opt->val != NULL && conn_opt->val[0] != '\0' && - strcmp(conn_opt->keyword, "dbname") != 0) - { - keywords[i] = conn_opt->keyword; - values[i] = conn_opt->val; - i++; - } - } - } - else - { - keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); - values = pg_malloc0((argcount + 1) * sizeof(*values)); - } - - if (pghost) - { - keywords[i] = "host"; - values[i] = pghost; - i++; - } - if (pgport) - { - keywords[i] = "port"; - values[i] = pgport; - i++; - } - if (pguser) - { - keywords[i] = "user"; - values[i] = pguser; - i++; - } - if (password) - { - keywords[i] = "password"; - values[i] = password; - i++; - } - if (dbname) - { - keywords[i] = "dbname"; - values[i] = dbname; - i++; - } - keywords[i] = "fallback_application_name"; - values[i] = progname; - i++; - - new_pass = false; - conn = PQconnectdbParams(keywords, values, true); - - if (!conn) - pg_fatal("could not connect to database \"%s\"", dbname); - - if (PQstatus(conn) == CONNECTION_BAD && - PQconnectionNeedsPassword(conn) && - !password && - prompt_password != TRI_NO) - { - PQfinish(conn); - password = simple_prompt("Password: ", false); - new_pass = true; - } - } while (new_pass); - - /* check to see that the backend connection was successfully made */ - if (PQstatus(conn) == CONNECTION_BAD) - { - if (fail_on_error) - pg_fatal("%s", PQerrorMessage(conn)); - else - { - PQfinish(conn); - - free(keywords); - free(values); - PQconninfoFree(conn_opts); - - return NULL; - } - } - - /* - * Ok, connected successfully. Remember the options used, in the form of a - * connection string. - */ - connstr = constructConnStr(keywords, values); - - free(keywords); - free(values); - PQconninfoFree(conn_opts); - - /* Check version */ - remoteversion_str = PQparameterStatus(conn, "server_version"); - if (!remoteversion_str) - pg_fatal("could not get server version"); - server_version = PQserverVersion(conn); - if (server_version == 0) - pg_fatal("could not parse server version \"%s\"", - remoteversion_str); - - my_version = PG_VERSION_NUM; - - /* - * We allow the server to be back to 9.2, and up to any minor release of - * our own major version. (See also version check in pg_dump.c.) - */ - if (my_version != server_version - && (server_version < 90200 || - (server_version / 100) > (my_version / 100))) - { - pg_log_error("aborting because of server version mismatch"); - pg_log_error_detail("server version: %s; %s version: %s", - remoteversion_str, progname, PG_VERSION); - exit_nicely(1); - } - - PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL)); - - return conn; -} - -/* ---------- - * Construct a connection string from the given keyword/value pairs. It is - * used to pass the connection options to the pg_dump subprocess. - * - * The following parameters are excluded: - * dbname - varies in each pg_dump invocation - * password - it's not secure to pass a password on the command line - * fallback_application_name - we'll let pg_dump set it - * ---------- - */ -static char * -constructConnStr(const char **keywords, const char **values) -{ - PQExpBuffer buf = createPQExpBuffer(); - char *connstr; - int i; - bool firstkeyword = true; - - /* Construct a new connection string in key='value' format. */ - for (i = 0; keywords[i] != NULL; i++) - { - if (strcmp(keywords[i], "dbname") == 0 || - strcmp(keywords[i], "password") == 0 || - strcmp(keywords[i], "fallback_application_name") == 0) - continue; - - if (!firstkeyword) - appendPQExpBufferChar(buf, ' '); - firstkeyword = false; - appendPQExpBuffer(buf, "%s=", keywords[i]); - appendConnStrVal(buf, values[i]); - } - - connstr = pg_strdup(buf->data); - destroyPQExpBuffer(buf); - return connstr; -} - -/* - * Run a query, return the results, exit program on failure. - */ -static PGresult * -executeQuery(PGconn *conn, const char *query) -{ - PGresult *res; - - pg_log_info("executing %s", query); - - res = PQexec(conn, query); - if (!res || - PQresultStatus(res) != PGRES_TUPLES_OK) - { - pg_log_error("query failed: %s", PQerrorMessage(conn)); - pg_log_error_detail("Query was: %s", query); - PQfinish(conn); - exit_nicely(1); - } - - return res; -} - /* * As above for a SQL command (which returns nothing). */ @@ -1994,3 +1859,58 @@ read_dumpall_filters(const char *filename, SimpleStringList *pattern) filter_free(&fstate); } + +/* + * create_or_open_dir + * + * This will create a new directory with given name. If there is already same + * empty directory exist, then use it. + */ +static void +create_or_open_dir(const char *dirname) +{ + struct stat st; + bool is_empty = false; + + /* we accept an empty existing directory */ + if (stat(dirname, &st) == 0 && S_ISDIR(st.st_mode)) + { + DIR *dir = opendir(dirname); + + if (dir) + { + struct dirent *d; + + is_empty = true; + + while (errno = 0, (d = readdir(dir))) + { + if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0) + { + is_empty = false; + break; + } + } + + if (errno) + pg_fatal("could not read directory \"%s\": %m", + dirname); + + if (closedir(dir)) + pg_fatal("could not close directory \"%s\": %m", + dirname); + } + + if(!is_empty) + { + pg_log_error("directory \"%s\" exists but is not empty", dirname); + pg_log_error_hint("If you want to dump data on this directory, either remove or empty " + "this directory \"%s\" or run %s " + "with an argument other than \"%s\".", + dirname, progname, dirname); + exit_nicely(1); + } + } + else if (mkdir(dirname, 0700) < 0) + pg_fatal("could not create directory \"%s\": %m", dirname); +} diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 88ae39d938..79f61395ae 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -2,7 +2,7 @@ * * pg_restore.c * pg_restore is an utility extracting postgres database definitions - * from a backup archive created by pg_dump using the archiver + * from a backup archive created by pg_dump/pg_dumpall using the archiver * interface. * * pg_restore will read the backup archive and @@ -41,27 +41,67 @@ #include "postgres_fe.h" #include +#include #ifdef HAVE_TERMIOS_H #include #endif +#include "common/connect.h" +#include "compress_io.h" +#include "common/string.h" +#include "common_dumpall_restore.h" #include "fe_utils/option_utils.h" +#include "fe_utils/string_utils.h" #include "filter.h" #include "getopt_long.h" #include "parallel.h" +#include "pg_backup_archiver.h" #include "pg_backup_utils.h" +typedef struct SimpleDatabaseOidListCell +{ + struct SimpleDatabaseOidListCell *next; + Oid db_oid; + const char *db_name; +} SimpleDatabaseOidListCell; + +typedef struct SimpleDatabaseOidList +{ + SimpleDatabaseOidListCell *head; + SimpleDatabaseOidListCell *tail; +} SimpleDatabaseOidList; + +static void +simple_db_oid_list_append(SimpleDatabaseOidList *list, Oid db_oid, const char *dbname); + static void usage(const char *progname); static void read_restore_filters(const char *filename, RestoreOptions *opts); +static bool IsFileExistsInDirectory(const char *dir, const char *filename); +static bool restoreOneDatabase(const char *inputFileSpec, + RestoreOptions *opts, int numWorkers, bool append_data); +static int ReadOneStatement(StringInfo inBuf, FILE *pfile); +static int restoreAllDatabases(PGconn *conn, const char *dumpdirpath, + SimpleStringList db_exclude_patterns, RestoreOptions *opts, int numWorkers); +static void execute_global_sql_commands(PGconn *conn, const char *dumpdirpath, + const char *outfile); +static int filter_dbnames_for_restore(PGconn *conn, + SimpleDatabaseOidList *dbname_oid_list, SimpleStringList db_exclude_patterns); +static int get_dbname_oid_list_from_mfile(const char *dumpdirpath, + SimpleDatabaseOidList *dbname_oid_list); +static void simple_db_oid_list_append(SimpleDatabaseOidList *list, Oid db_oid, + const char *dbname); +static bool is_full_pattern(PGconn *conn, const char *str, const char *ptrn); +static void simple_string_full_list_delete(SimpleStringList *list); +static void simple_db_oid_full_list_delete(SimpleDatabaseOidList *list); +static void simple_db_oid_list_delete(SimpleDatabaseOidList *list, + SimpleDatabaseOidListCell *cell, SimpleDatabaseOidListCell *prev); int main(int argc, char **argv) { RestoreOptions *opts; int c; - int exit_code; int numWorkers = 1; - Archive *AH; char *inputFileSpec; static int disable_triggers = 0; static int enable_row_security = 0; @@ -77,11 +117,14 @@ main(int argc, char **argv) static int strict_names = 0; bool data_only = false; bool schema_only = false; + SimpleStringList db_exclude_patterns = {NULL, NULL}; + bool globals_only = false; struct option cmdopts[] = { {"clean", 0, NULL, 'c'}, {"create", 0, NULL, 'C'}, {"data-only", 0, NULL, 'a'}, + {"globals-only", 0, NULL, 'g'}, {"dbname", 1, NULL, 'd'}, {"exit-on-error", 0, NULL, 'e'}, {"exclude-schema", 1, NULL, 'N'}, @@ -128,6 +171,7 @@ main(int argc, char **argv) {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, {"filter", required_argument, NULL, 4}, + {"exclude-database", required_argument, NULL, 6}, {NULL, 0, NULL, 0} }; @@ -156,7 +200,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", + while ((c = getopt_long(argc, argv, "aAcCd:ef:F:gh:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", cmdopts, NULL)) != -1) { switch (c) @@ -183,11 +227,14 @@ main(int argc, char **argv) if (strlen(optarg) != 0) opts->formatName = pg_strdup(optarg); break; + case 'g': + /* restore only global.dat file from directory */ + globals_only = true; + break; case 'h': if (strlen(optarg) != 0) opts->cparams.pghost = pg_strdup(optarg); break; - case 'j': /* number of restore jobs */ if (!option_parse_int(optarg, "-j/--jobs", 1, PG_MAX_JOBS, @@ -302,6 +349,10 @@ main(int argc, char **argv) exit(1); opts->exit_on_error = true; break; + case 6: + /* list of databases patterns those needs to skip while restoring */ + simple_string_list_append(&db_exclude_patterns, optarg); + break; default: /* getopt_long already emitted a complaint */ @@ -329,6 +380,13 @@ main(int argc, char **argv) if (!opts->cparams.dbname && !opts->filename && !opts->tocSummary) pg_fatal("one of -d/--dbname and -f/--file must be specified"); + if (db_exclude_patterns.head != NULL && globals_only) + { + pg_log_error("option --exclude-database cannot be used together with -g/--globals-only"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit_nicely(1); + } + /* Should get at most one of -d and -f, else user is confused */ if (opts->cparams.dbname) { @@ -383,28 +441,80 @@ main(int argc, char **argv) if (opts->formatName) { - switch (opts->formatName[0]) + opts->format = parseDumpFormat(opts->formatName); + + /* Plain format is not supported for pg_restore. */ + if (opts->format == archNull) { - case 'c': - case 'C': - opts->format = archCustom; - break; + pg_fatal("unrecognized archive format \"%s\"; please specify \"c\", \"d\", or \"t\"", + opts->formatName); + } + } - case 'd': - case 'D': - opts->format = archDirectory; - break; + /* + * If toc.dat file does not present in current path, then check for + * global.dat. If global.dat file is present, then restore all the + * databases from map.dat(if exist) file list and skip restoring for + * --exclude-database patterns. + */ + if (inputFileSpec != NULL && + !IsFileExistsInDirectory(inputFileSpec, "toc.dat")) + { + /* If global.dat is exist, then process it. */ + if (IsFileExistsInDirectory(pg_strdup(inputFileSpec), "global.dat")) + { + PGconn *conn = NULL; /* Connection to restore global sql commands. */ + int exit_code = 0; - case 't': - case 'T': - opts->format = archTar; - break; + /* Connect to database to execute global sql commands from global.dat file. */ + if (opts->cparams.dbname) + { + conn = connectDatabase(opts->cparams.dbname, NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, false, + progname, NULL, NULL); - default: - pg_fatal("unrecognized archive format \"%s\"; please specify \"c\", \"d\", or \"t\"", - opts->formatName); - } - } + if (!conn) + pg_fatal("could not connect to database \"%s\"", opts->cparams.dbname); + } + + /* Open global.dat file and execute/append all the global sql commands. */ + execute_global_sql_commands(conn, inputFileSpec, opts->filename); + + /* If globals-only, then return from here. */ + if (globals_only) + { + if (conn) + PQfinish(conn); + pg_log_info("databases restoring is skipped as -g/--globals-only option is specified"); + } + else + { + /* Now restore all the databases from map.dat file. */ + exit_code = restoreAllDatabases(conn, inputFileSpec, db_exclude_patterns, + opts, numWorkers); + } + + /* Free db pattern list. */ + simple_string_full_list_delete(&db_exclude_patterns); + + return exit_code; + }/* end if */ + }/* end if */ + + return restoreOneDatabase(inputFileSpec, opts, numWorkers, false); +} + +/* + * restoreOneDatabase + * + * This will restore one database using toc.dat file. + */ +static bool +restoreOneDatabase(const char *inputFileSpec, RestoreOptions *opts, + int numWorkers, bool append_data) +{ + Archive *AH; + bool exit_code; AH = OpenArchive(inputFileSpec, opts->format); @@ -431,11 +541,11 @@ main(int argc, char **argv) AH->numWorkers = numWorkers; if (opts->tocSummary) - PrintTOCSummary(AH); + PrintTOCSummary(AH, append_data); else { ProcessArchiveRestoreOptions(AH); - RestoreArchive(AH); + RestoreArchive(AH, append_data); } /* done, print a summary of ignored errors */ @@ -471,6 +581,7 @@ usage(const char *progname) printf(_(" -c, --clean clean (drop) database objects before recreating\n")); printf(_(" -C, --create create the target database\n")); printf(_(" -e, --exit-on-error exit on error, default is to continue\n")); + printf(_(" -g, --globals-only restore only global objects, no databases\n")); printf(_(" -I, --index=NAME restore named index\n")); printf(_(" -j, --jobs=NUM use this many parallel jobs to restore\n")); printf(_(" -L, --use-list=FILENAME use table of contents from this file for\n" @@ -483,6 +594,7 @@ usage(const char *progname) printf(_(" -S, --superuser=NAME superuser user name to use for disabling triggers\n")); printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n")); printf(_(" -T, --trigger=NAME restore named trigger\n")); + printf(_(" --exclude-database=PATTERN exclude databases whose name matches with pattern\n")); printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n")); printf(_(" -1, --single-transaction restore as a single transaction\n")); printf(_(" --disable-triggers disable triggers during data-only restore\n")); @@ -621,3 +733,542 @@ read_restore_filters(const char *filename, RestoreOptions *opts) filter_free(&fstate); } + +/* + * IsFileExistsInDirectory + * + * Returns true if file exist in current directory. + */ +static bool +IsFileExistsInDirectory(const char *dir, const char *filename) +{ + struct stat st; + char buf[MAXPGPATH]; + + if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH) + pg_fatal("directory name too long: \"%s\"", dir); + + return (stat(buf, &st) == 0 && S_ISREG(st.st_mode)); +} + +/* + * ReadOneStatement + * + * This will start reading from passed file pointer using fgetc and read till + * semicolon(sql statement terminator for global.sql file) + * + * EOF is returned if end-of-file input is seen; time to shut down. + */ + +static int +ReadOneStatement(StringInfo inBuf, FILE *pfile) +{ + int c; /* character read from getc() */ + + resetStringInfo(inBuf); + + /* + * Read characters until EOF or the appropriate delimiter is seen. + */ + while ((c = fgetc(pfile)) != EOF) + { + appendStringInfoChar(inBuf, (char) c); + + if (c == '\n') + { + if(inBuf->len > 1 && + inBuf->data[inBuf->len - 2] == ';') + break; + else + continue; + } + } + + /* No input before EOF signal means time to quit. */ + if (c == EOF && inBuf->len == 0) + return EOF; + + /* Add '\0' to make it look the same as message case. */ + appendStringInfoChar(inBuf, (char) '\0'); + + return 'Q'; +} + +/* + * filter_dbnames_for_restore + * + * This will remove names from all dblist that are given with exclude-database + * option. + * + * returns number of dbnames those will be restored. + */ +static int +filter_dbnames_for_restore(PGconn *conn, SimpleDatabaseOidList *dbname_oid_list, + SimpleStringList db_exclude_patterns) +{ + SimpleDatabaseOidListCell *dboid_cell = dbname_oid_list->head; + SimpleDatabaseOidListCell *dboidprecell = NULL; + int count_db = 0; + + /* Return 0 if there is no db to restore. */ + if (dboid_cell == NULL) + return 0; + + /* Process one by one all dbnames and if needs to skip restoring. */ + while (dboid_cell != NULL) + { + bool skip_db_restore = false; + SimpleDatabaseOidListCell *next = dboid_cell->next; + + /* Now match this dbname with exclude-database list. */ + for (SimpleStringListCell *celldb = db_exclude_patterns.head; celldb; celldb = celldb->next) + { + if ((conn && is_full_pattern(conn, dboid_cell->db_name, celldb->val)) || + (!conn && pg_strcasecmp(dboid_cell->db_name, celldb->val) == 0)) + { + /* + * As we need to skip this dbname so set flag to remove it from + * list. + * + * Note: we can't remove this pattern from skip list as we + * might have multiple database name with this same pattern. + */ + skip_db_restore = true; + break; + } + } + + /* Increment count if db needs to be restored. */ + if (skip_db_restore) + { + pg_log_info("excluding database \"%s\"", dboid_cell->db_name); + simple_db_oid_list_delete(dbname_oid_list, dboid_cell, dboidprecell); + } + else + { + count_db++; /* Increment db couter. */ + dboidprecell = dboid_cell; + } + + /* Process next dbname from dbname list. */ + dboid_cell = next; + } + + return count_db; +} + +/* + * get_dbname_oid_list_from_mfile + * + * Open map.dat file and read line by line and then prepare a list of database + * names and correspoding db_oid. + * + * Returns, total number of database names in map.dat file. + */ +static int +get_dbname_oid_list_from_mfile(const char *dumpdirpath, SimpleDatabaseOidList *dbname_oid_list) +{ + FILE *pfile; + char map_file_path[MAXPGPATH]; + char line[MAXPGPATH]; + int count = 0; + + if (!IsFileExistsInDirectory(pg_strdup(dumpdirpath), "map.dat")) + { + pg_log_info("map.dat file is not present in dump of pg_dumpall, so nothing to restore."); + return 0; + } + + snprintf(map_file_path, MAXPGPATH, "%s/map.dat", dumpdirpath); + + /* Open map.dat file. */ + pfile = fopen(map_file_path, PG_BINARY_R); + + if (pfile == NULL) + pg_fatal("could not open map.dat file: \"%s\"", map_file_path); + + /* Append all the dbname and db_oid to the list. */ + while((fgets(line, MAXPGPATH, pfile)) != NULL) + { + Oid db_oid; + char db_oid_str[MAXPGPATH + 1]; + char dbname[MAXPGPATH + 1]; + + /* Extract dboid. */ + sscanf(line, "%u" , &db_oid); + sscanf(line, "%s" , db_oid_str); + + /* Now copy dbname. */ + strcpy(dbname, line + strlen(db_oid_str) + 1); + + /* Remove \n from dbanme. */ + dbname[strlen(dbname) - 1] = '\0'; + + pg_log_info("found dbname as : \"%s\" and db_oid:%u in map.dat file while restoring", dbname, db_oid); + + /* Report error if file has any corrupted data. */ + if (!OidIsValid(db_oid) || strlen(dbname) == 0) + pg_fatal("invalid entry in map.dat file at line : %d", count + 1); + + /* + * XXX : before adding dbname into list, we can verify that this db + * needs to skipped for restore or not but as of now, we are making + * a list of all the databases. + */ + simple_db_oid_list_append(dbname_oid_list, db_oid, dbname); + count++; + } + + /* Close map.dat file. */ + fclose(pfile); + + return count; +} + +/* + * restoreAllDatabases + * + * This will restore databases those dumps are present in + * directory based on map.dat file mapping. + * + * This will skip restoring for databases that are specified with + * exclude-database option. + */ +static int +restoreAllDatabases(PGconn *conn, const char *dumpdirpath, + SimpleStringList db_exclude_patterns, RestoreOptions *opts, + int numWorkers) +{ + SimpleDatabaseOidList dbname_oid_list = {NULL, NULL}; + SimpleDatabaseOidListCell *dboid_cell; + int exit_code = 0; + int num_db_restore; + int num_total_db; + + num_total_db = get_dbname_oid_list_from_mfile(dumpdirpath, &dbname_oid_list); + + /* If map.dat has no entry, return from here. */ + if (dbname_oid_list.head == NULL) + return 0; + + pg_log_info("found total %d database names in map.dat file", num_total_db); + + if (!conn) + { + pg_log_info("trying to connect postgres database to dump into out file"); + + conn = connectDatabase("postgres", NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, false, + progname, NULL, NULL); + + /* Try with template1. */ + if (!conn) + { + pg_log_info("trying to connect template1 database as failed to connect to postgres to dump into out file"); + + conn = connectDatabase("template1", NULL, opts->cparams.pghost, + opts->cparams.pgport, opts->cparams.username, TRI_DEFAULT, false, + progname, NULL, NULL); + + if (!conn) + pg_log_info("there is no database connection so consider pattern as simple name for exclude-database"); + } + } + + /* + * TODO: To skip databases, we need to make a design. + * + * Skip any explicitly excluded database. If there is no database + * connection, then just consider pattern as simple name to compare. + */ + num_db_restore = filter_dbnames_for_restore(conn, &dbname_oid_list, + db_exclude_patterns); + + /* Close the db connection as we are done globals and patterns. */ + if (conn) + PQfinish(conn); + + /* Exit if no db needs to be restored. */ + if (dbname_oid_list.head == NULL) + return 0; + + pg_log_info("needs to restore %d databases out of %d databases", num_db_restore, num_total_db); + + /* + * To restore multiple databases, -C (create database) option should be specified + * or all databases should be created before pg_restore. + */ + if (opts->createDB != 1) + pg_log_info("restoring dump of pg_dumpall without -C option, there might be multiple databases in directory."); + + /* TODO: MAX_ON_EXIT_NICELY is 100 now... max AH handle register on exit .*/ + if (num_db_restore > 100) + { + simple_db_oid_full_list_delete(&dbname_oid_list); + pg_fatal("cound not restore more than 100 databases by single pg_restore, here total db:%d", num_db_restore); + } + + /* + * XXX: TODO till now, we made a list of databases, those needs to be restored + * after skipping names of exclude-database. Now we can launch parallel + * workers to restore these databases. + */ + dboid_cell = dbname_oid_list.head; + + while(dboid_cell != NULL) + { + char subdirpath[MAXPGPATH]; + int dbexit_code; + + /* + * We need to reset override_dbname so that objects can be restored into + * already created database. (used with -d/--dbname option) + */ + if (opts->cparams.override_dbname) + { + pfree(opts->cparams.override_dbname); + opts->cparams.override_dbname = NULL; + } + + snprintf(subdirpath, MAXPGPATH, "%s/databases/%u", dumpdirpath, dboid_cell->db_oid); + + pg_log_info("restoring database \"%s\"", dboid_cell->db_name); + + dbexit_code = restoreOneDatabase(subdirpath, opts, numWorkers, true); + + /* Store exit_code to report it back. */ + if (exit_code == 0 && dbexit_code != 0) + exit_code = dbexit_code; + + dboid_cell = dboid_cell->next; + } /* end while */ + + /* Log number of processed databases.*/ + pg_log_info("number of restored databases are %d", num_db_restore); + + /* Free dbname and dboid list. */ + simple_db_oid_full_list_delete(&dbname_oid_list); + + return exit_code; +} + +/* + * execute_global_sql_commands + * + * This will open global.dat file and will execute all global sql commands one + * by one statement. + * Semicolon is considered as statement terminator. If outfile is passed, then + * this will copy all sql commands into outfile rather then executing them. + */ +static void +execute_global_sql_commands(PGconn *conn, const char *dumpdirpath, const char *outfile) +{ + char global_file_path[MAXPGPATH]; + PGresult *result; + StringInfoData sqlstatement; + FILE *pfile; + + snprintf(global_file_path, MAXPGPATH, "%s/global.dat", dumpdirpath); + + /* Now open global.dat file. */ + pfile = fopen(global_file_path, PG_BINARY_R); + + if (pfile == NULL) + pg_fatal("could not open global.dat file: \"%s\"", global_file_path); + + /* + * If outfile is given, then just copy all global.dat file data into + * outfile. + */ + if (outfile) + { + char out_file_path[MAXPGPATH]; + FILE *ofile; + int c; + + snprintf(out_file_path, MAXPGPATH, "%s", outfile); + + ofile = fopen(out_file_path, PG_BINARY_W); + + if (ofile == NULL) + { + fclose(pfile); + pg_fatal("could not open file: \"%s\"", out_file_path); + } + + /* Now append global.dat inot outfile. */ + while ((c = fgetc(pfile)) != EOF) + { + fputc(c, ofile); + } + + fclose(pfile); + fclose(ofile); + return; + } + + /* Init sqlstatement to append commands */ + initStringInfo(&sqlstatement); + + /* Process file till EOF and execute sql statements */ + while (ReadOneStatement(&sqlstatement, pfile) != EOF) + { + result = PQexec(conn, sqlstatement.data); + + switch (PQresultStatus(result)) + { + case PGRES_COMMAND_OK: + case PGRES_TUPLES_OK: + case PGRES_EMPTY_QUERY: + break; + default: + pg_log_error("could not execute query: \"%s\" \nCommand was: \"%s\"", PQerrorMessage(conn), sqlstatement.data); + } + PQclear(result); + } + + fclose(pfile); +} + +/* + * simple_db_oid_list_append + * + * appends a node to the list in the end. + */ +static void +simple_db_oid_list_append(SimpleDatabaseOidList *list, Oid db_oid, const char *dbname) +{ + SimpleDatabaseOidListCell *cell; + + cell = pg_malloc_object(SimpleDatabaseOidListCell); + + cell->next = NULL; + cell->db_oid = db_oid; + cell->db_name = pg_strdup(dbname); + + if (list->tail) + list->tail->next = cell; + else + list->head = cell; + list->tail = cell; +} + +/* + * simple_db_oid_full_list_delete + * + * delete all cell from dbname and dboid list. + */ +static void +simple_db_oid_full_list_delete(SimpleDatabaseOidList *list) +{ + SimpleDatabaseOidListCell *cell = list->head; + SimpleDatabaseOidListCell *nextcell = NULL; + + while (cell) + { + nextcell = cell->next; + pfree (cell); + cell = nextcell; + } + + list->head = NULL; + list->tail = NULL; +} + +/* + * simple_string_full_list_delete + * + * delete all cell from string list. + */ +static void +simple_string_full_list_delete(SimpleStringList *list) +{ + SimpleStringListCell *cell = list->head; + SimpleStringListCell *cellnext = NULL; + + while (cell) + { + cellnext = cell->next; + pfree(cell); + cell = cellnext; + } + + list->head = NULL; + list->tail = NULL; +} + +/* + * simple_db_oid_list_delete + * + * delete cell from database and oid list. + */ +static void +simple_db_oid_list_delete(SimpleDatabaseOidList *list, SimpleDatabaseOidListCell *cell, + SimpleDatabaseOidListCell *prev) +{ + if (prev == NULL) + { + list->head = cell->next; + pfree(cell); + } + else + { + prev->next = cell->next; + pfree(cell); + } +} + +/* + * is_full_pattern + * + * This uses substring function to make 1st string from pattern. + * Outstring of substring function is compared with 1st string to + * validate this pattern. + * + * Returns true if 1st string can be constructed from given pattern. + * + */ +static bool +is_full_pattern(PGconn *conn, const char *str, const char *ptrn) +{ + PQExpBuffer query; + PGresult *result; + + query = createPQExpBuffer(); + + printfPQExpBuffer(query, + "SELECT substring ( " + " '%s' , " + " '%s' ) ", str, ptrn); + + result = executeQuery(conn, query->data); + + if (PQresultStatus(result) == PGRES_TUPLES_OK) + { + if (PQntuples(result) == 1) + { + const char *outstr = NULL; + + /* + * If output string of substring function is matches with str, then + * we can construct str from pattern. + */ + if (!PQgetisnull(result, 0, 0)) + outstr = PQgetvalue(result, 0, 0); + + if (outstr && pg_strcasecmp(outstr, str) == 0) + { + PQclear(result); + destroyPQExpBuffer(query); + return true; + } + } + } + else + pg_log_error("could not execute query: \"%s\" \nCommand was: \"%s\"", PQerrorMessage(conn), query->data); + + PQclear(result); + destroyPQExpBuffer(query); + + return false; +} diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl old mode 100644 new mode 100755 index 214240f1ae..20ddf3646a --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -219,6 +219,13 @@ command_fails_like( 'pg_restore: options -C\/--create and -1\/--single-transaction cannot be used together' ); +command_fails_like( + [ 'pg_restore', '-p', 'xxx', '-f', 'xxx', + "--exclude-database=grabadge", + '--globals-only' ], + qr/\Qpg_restore: error: option --exclude-database cannot be used together with -g\/--globals-only\E/, + 'pg_restore: option --exclude-database cannot be used together with -g/--globals-only'); + # also fails for -r and -t, but it seems pointless to add more tests for those. command_fails_like( [ 'pg_dumpall', '--exclude-database=foo', '--globals-only' ], @@ -226,4 +233,8 @@ command_fails_like( 'pg_dumpall: option --exclude-database cannot be used together with -g/--globals-only' ); +command_fails_like( + [ 'pg_dumpall', '--format', 'x' ], + qr/\Qpg_dumpall: error: unrecognized archive format "x";\E/, + 'pg_dumpall: unrecognized archive format'); done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a2644a2e65..96f460d2e3 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2673,6 +2673,8 @@ ShutdownMode SignTSVector SimpleActionList SimpleActionListCell +SimpleDatabaseOidList +SimpleDatabaseOidListCell SimpleEcontextStackEntry SimpleOidList SimpleOidListCell -- 2.43.0