From 097f92364d87768ed7ca5b47dbf80d78e5073777 Mon Sep 17 00:00:00 2001 From: Shubham Khanna Date: Wed, 22 Jan 2025 12:03:12 +0530 Subject: [PATCH v9] Enhance 'pg_createsubscriber' to fetch and append all databases This patch enhances the 'pg_createsubscriber' utility by adding the '--all-databases' option. When '--all-databases' is specified, the tool queries the source server (publisher) for all databases and creates subscriptions on the target server (subscriber) for databases with matching names. This simplifies the process of converting a physical standby to a logical subscriber, particularly during upgrades. The options '--database', '--publication', '--subscription', and '--replication-slot' cannot be used when '--all-databases' is specified. --- doc/src/sgml/ref/pg_createsubscriber.sgml | 48 +++++- src/bin/pg_basebackup/pg_createsubscriber.c | 104 ++++++++++- .../t/040_pg_createsubscriber.pl | 162 +++++++++++++++++- 3 files changed, 305 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml index d56487fe2c..4627c931e2 100644 --- a/doc/src/sgml/ref/pg_createsubscriber.sgml +++ b/doc/src/sgml/ref/pg_createsubscriber.sgml @@ -20,6 +20,27 @@ PostgreSQL documentation + + pg_createsubscriber + option + + + + + + + + + + datadir + + + + + connstr + + + pg_createsubscriber option @@ -87,6 +108,22 @@ PostgreSQL documentation command-line arguments: + + + + + + Automatically fetch all non-template databases from the source server + and create subscriptions for databases with the same names on the + target server. + Subscription names, publication names, and replication slot names are + automatically generated. Cannot be used together with + , , + or . + + + + @@ -94,7 +131,7 @@ PostgreSQL documentation The name of the database in which to create a subscription. Multiple databases can be selected by writing multiple - switches. + switches. Cannot be used together with . @@ -214,7 +251,8 @@ PostgreSQL documentation names must match the number of specified databases, otherwise an error is reported. The order of the multiple publication name switches must match the order of database switches. If this option is not specified, - a generated name is assigned to the publication name. + a generated name is assigned to the publication name. Cannot be used + together with . @@ -230,7 +268,8 @@ PostgreSQL documentation otherwise an error is reported. The order of the multiple replication slot name switches must match the order of database switches. If this option is not specified, the subscription name is assigned to the - replication slot name. + replication slot name. Cannot be used together with + . @@ -245,7 +284,8 @@ PostgreSQL documentation names must match the number of specified databases, otherwise an error is reported. The order of the multiple subscription name switches must match the order of database switches. If this option is not specified, - a generated name is assigned to the subscription name. + a generated name is assigned to the subscription name. Cannot be used + together with . diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c index 37fdf150b4..4d87bd0fd0 100644 --- a/src/bin/pg_basebackup/pg_createsubscriber.c +++ b/src/bin/pg_basebackup/pg_createsubscriber.c @@ -43,6 +43,7 @@ struct CreateSubscriberOptions SimpleStringList sub_names; /* list of subscription names */ SimpleStringList replslot_names; /* list of replication slot names */ int recovery_timeout; /* stop recovery after this time */ + bool all; /* --all option was specified */ }; struct LogicalRepInfo @@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn, const struct LogicalRepInfo *dbinfo); static void drop_existing_subscriptions(PGconn *conn, const char *subname, const char *dbname); +static void fetch_source_databases(struct CreateSubscriberOptions *opt); #define USEC_PER_SEC 1000000 #define WAIT_INTERVAL 1 /* 1 second */ @@ -220,6 +222,7 @@ usage(void) printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nOptions:\n")); + printf(_(" -a, --all create subscriptions for all non-template source databases\n")); printf(_(" -d, --database=DBNAME database in which to create a subscription\n")); printf(_(" -D, --pgdata=DATADIR location for the subscriber data directory\n")); printf(_(" -n, --dry-run dry run, just show what would be done\n")); @@ -1879,11 +1882,62 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo) destroyPQExpBuffer(str); } +/* + * If --all is specified, fetch a list of all user-created databases from the + * source server. Internally, this is treated as if the user specified multiple + * --database options, one for each source database. + */ +static void +fetch_source_databases(struct CreateSubscriberOptions *opt) +{ + PGconn *conn; + PGresult *res; + int num_rows; + + /* Establish a connection to the PostgreSQL server */ + conn = connect_database(opt->pub_conninfo_str, true); + + res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true"); + + /* Check for errors during query execution */ + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res)); + PQclear(res); + disconnect_database(conn, true); + } + + /* Process the query result */ + num_rows = PQntuples(res); + for (int i = 0; i < num_rows; i++) + { + const char *dbname = PQgetvalue(res, i, 0); + + simple_string_list_append(&opt->database_names, dbname); + + /* Increment num_dbs to reflect multiple --database options */ + num_dbs++; + } + + /* Error if no databases were found on the source server */ + if (num_rows == 0) + { + pg_log_error("no suitable databases found on the source server"); + pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server."); + PQclear(res); + disconnect_database(conn, true); + } + + PQclear(res); + disconnect_database(conn, false); +} + int main(int argc, char **argv) { static struct option long_options[] = { + {"all", no_argument, NULL, 'a'}, {"database", required_argument, NULL, 'd'}, {"pgdata", required_argument, NULL, 'D'}, {"dry-run", no_argument, NULL, 'n'}, @@ -1946,6 +2000,7 @@ main(int argc, char **argv) opt.socket_dir = NULL; opt.sub_port = DEFAULT_SUB_PORT; opt.sub_username = NULL; + opt.all = false; opt.database_names = (SimpleStringList) { 0 @@ -1968,11 +2023,14 @@ main(int argc, char **argv) get_restricted_token(); - while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v", + while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v", long_options, &option_index)) != -1) { switch (c) { + case 'a': + opt.all = true; + break; case 'd': if (!simple_string_list_member(&opt.database_names, optarg)) { @@ -2057,6 +2115,38 @@ main(int argc, char **argv) } } + if (opt.all) + { + if (num_dbs > 0) + { + pg_log_error("%s cannot be used with %s", "--all", "--database"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + if (num_pubs > 0) + { + pg_log_error("%s cannot be used with %s", "--all", "--publication"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + if (num_replslots > 0) + { + pg_log_error("%s cannot be used with %s", "--all", "--replication-slot"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + if (num_subs > 0) + { + pg_log_error("%s cannot be used with %s", "--all", "--subscription"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + } + + /* Any non-option arguments? */ if (optind < argc) { @@ -2110,14 +2200,20 @@ main(int argc, char **argv) pg_log_info("validating subscriber connection string"); sub_base_conninfo = get_sub_conninfo(&opt); + /* + * Fetch all databases from the source (publisher) if --all is specified. + */ + if (opt.all) + fetch_source_databases(&opt); + if (opt.database_names.head == NULL) { pg_log_info("no database was specified"); /* - * If --database option is not provided, try to obtain the dbname from - * the publisher conninfo. If dbname parameter is not available, error - * out. + * If neither --database nor --all option is provided, try to obtain + * the dbname from the publisher conninfo. If dbname parameter is not + * available, error out. */ if (dbname_conninfo) { diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl index c8dbdb7e9b..837a2f1f86 100644 --- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl +++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl @@ -371,6 +371,165 @@ command_ok( ], 'run pg_createsubscriber without --databases'); +# run pg_createsubscriber with '--all' and '--database' and verify the +# failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--database' => $db1, + ], + qr/--all cannot be used with --database/, + 'fail if --all is used with --database'); + +# run pg_createsubscriber with '--database' and '--all' and verify the +# failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--database' => $db1, + '--all', + ], + qr/--all cannot be used with --database/, + 'fail if --all is used with --database'); + +# run pg_createsubscriber with '--database' and '--all' without '--dry-run' +# and verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--database' => $db1, + '--all', + ], + qr/--all cannot be used with --database/, + 'fail if --all is used with --database'); + +# run pg_createsubscriber with '--publication' and '--all' and verify +# the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--publication' => 'pub1', + ], + qr/--all cannot be used with --publication/, + 'fail if --all is used with --publication'); + +# run pg_createsubscriber with '--replication-slot' and '--all' and +# verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--replication-slot' => 'replslot1', + '--all', + ], + qr/--all cannot be used with --replication-slot/, + 'fail if --all is used with --replication-slot'); + +# run pg_createsubscriber with '--subscription' and '--all' and +# verify the failure +command_fails_like( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + '--subscription' => 'sub1', + ], + qr/--all cannot be used with --subscription/, + 'fail if --all is used with --subscription'); + +# run pg_createsubscriber with '--all' option +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--dry-run', + '--pgdata' => $node_s->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s->host, + '--subscriber-port' => $node_s->port, + '--all', + ], + 'run pg_createsubscriber with --all'); + +# Set up node S1 as standby linking to node P +$node_p->backup('backup_3'); +my $node_s1 = PostgreSQL::Test::Cluster->new('node_s1'); +$node_s1->init_from_backup($node_p, 'backup_3', has_streaming => 1); +$node_s1->append_conf( + 'postgresql.conf', qq[ +primary_conninfo = '$pconnstr dbname=postgres' +hot_standby_feedback = on +]); +$node_s1->set_standby_mode(); + +# run pg_createsubscriber with '--all' option without '--dry-run' +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_s1->data_dir, + '--publisher-server' => $node_p->connstr($db1), + '--socketdir' => $node_s1->host, + '--subscriber-port' => $node_s1->port, + '--all', + ], + 'run pg_createsubscriber with --all'); + +$node_s1->start; + +# Verify that only user databases got subscriptions (not template databases) +my @user_dbs = ($db1, $db2); +foreach my $dbname (@user_dbs) +{ + my $sub_exists = + $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;"); + is($sub_exists, '3', "Subscription created successfully for $dbname"); +} + +# Verify replication is working +$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');"); + +$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1"); +is( $result, qq(first row +replication test +second row), "replication successful in $db1"); + +$node_s1->stop; # Run pg_createsubscriber on node S. --verbose is used twice # to show more information. command_ok( @@ -431,8 +590,9 @@ $result = $node_s->safe_psql($db1, is($result, qq(0), 'failover slot was removed'); # Check result in database $db1 -$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1'); +$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1 ORDER BY 1'); is( $result, qq(first row +replication test second row third row), "logical replication works in database $db1"); -- 2.34.1