From 3240552a0e8853cb69191da760cdb34069d83da5 Mon Sep 17 00:00:00 2001 From: Shubham Khanna Date: Wed, 22 Jan 2025 12:03:12 +0530 Subject: [PATCH v1] Enhance 'pg_createsubscriber' to fetch and append all databases This patch introduces the ability for 'pg_createsubscriber' to automatically fetch all non-template databases from the publisher and append them to the subscription list. This improvement simplifies the creation of subscriptions for multiple databases by eliminating the need to specify each database manually. A new function 'fetch_and_append_all_databases' in 'pg_createsubscriber.c' that queries the publisher for all databases and adds them to the subscription options is added. --- doc/src/sgml/ref/pg_createsubscriber.sgml | 7 ++- src/bin/pg_basebackup/pg_createsubscriber.c | 60 +++++++++++++++++++ .../t/040_pg_createsubscriber.pl | 52 ++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml index 26b8e64a4e..0f7c441381 100644 --- a/doc/src/sgml/ref/pg_createsubscriber.sgml +++ b/doc/src/sgml/ref/pg_createsubscriber.sgml @@ -94,7 +94,12 @@ PostgreSQL documentation The name of the database in which to create a subscription. Multiple databases can be selected by writing multiple - switches. + switches. If this option is not provided, + pg_createsubscriber will automatically fetch + all non-template databases from the publisher and create subscriptions + for them. This is particularly useful in scenarios where the user wants + to convert a physical standby to a logical subscriber for + multiple databases, such as during an upgrade. diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c index faf18ccf13..db9031c377 100644 --- a/src/bin/pg_basebackup/pg_createsubscriber.c +++ b/src/bin/pg_basebackup/pg_createsubscriber.c @@ -106,6 +106,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn, const struct LogicalRepInfo *dbinfo); static void drop_existing_subscriptions(PGconn *conn, const char *subname, const char *dbname); +static void fetch_and_append_all_databases(struct CreateSubscriberOptions *opt); #define USEC_PER_SEC 1000000 #define WAIT_INTERVAL 1 /* 1 second */ @@ -1860,6 +1861,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo) destroyPQExpBuffer(str); } +/* Placeholder function to fetch all databases from the publisher */ +static void +fetch_and_append_all_databases(struct CreateSubscriberOptions *opt) +{ + PGconn *conn; + PGresult *res; + int num_rows; + const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false"; + + /* Establish a connection to the PostgreSQL server */ + conn = PQconnectdb(opt->pub_conninfo_str); + /* Check for connection errors */ + if (PQstatus(conn) != CONNECTION_OK) + { + pg_log_error("Connection to the PostgreSQL server failed: %s", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + + res = PQexec(conn, query); + + /* Check for errors during query execution */ + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("Failed to execute query on the PostgreSQL server: %s", PQerrorMessage(conn)); + PQclear(res); + PQfinish(conn); + exit(1); + } + + /* Process the query result */ + num_rows = PQntuples(res); + for (int i = 0; i < num_rows; i++) + { + const char *dbname = PQgetvalue(res, i, 0); + + simple_string_list_append(&opt->database_names, dbname); + num_dbs++; + } + + /* Check if any databases were added */ + if (opt->database_names.head == NULL) + { + pg_log_error("no database names could be fetched or specified"); + pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option."); + PQclear(res); + PQfinish(conn); + exit(1); + } + else + { + pg_log_info("databases were fetched and added to the subscription list"); + } + + PQclear(res); + PQfinish(conn); +} + int main(int argc, char **argv) { @@ -2115,6 +2174,7 @@ main(int argc, char **argv) progname); exit(1); } + fetch_and_append_all_databases(&opt); } /* Number of object names must match number of databases */ diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl index c8dbdb7e9b..eed4b18742 100644 --- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl +++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl @@ -368,6 +368,9 @@ command_ok( '--socketdir' => $node_s->host, '--subscriber-port' => $node_s->port, '--replication-slot' => 'replslot1', + '--replication-slot' => 'replslot2', + '--replication-slot' => 'replslot3', + '--replication-slot' => 'replslot4', ], 'run pg_createsubscriber without --databases'); @@ -448,10 +451,59 @@ my $sysid_s = $node_s->safe_psql('postgres', 'SELECT system_identifier FROM pg_control_system()'); ok($sysid_p != $sysid_s, 'system identifier was changed'); +# Set up node A as primary +my $node_a = PostgreSQL::Test::Cluster->new('node_a'); +my $aconnstr = $node_a->connstr; +$node_a->init(allows_streaming => 'logical'); +$node_a->append_conf('postgresql.conf', 'autovacuum = off'); +$node_a->start; + +# Set up node B as standby linking to node A +$node_a->backup('backup_3'); +my $node_b = PostgreSQL::Test::Cluster->new('node_b'); +$node_b->init_from_backup($node_a, 'backup_3', has_streaming => 1); +$node_b->append_conf( + 'postgresql.conf', qq[ +primary_conninfo = '$aconnstr dbname=postgres' +hot_standby_feedback = on +max_logical_replication_workers = 5 +]); +$node_b->set_standby_mode(); +$node_b->start; +# Ensure there are some user databases on the publisher +$node_a->safe_psql('postgres', 'CREATE DATABASE db3'); +$node_a->safe_psql('postgres', 'CREATE DATABASE db4'); +$node_a->safe_psql('postgres', 'CREATE DATABASE db5'); + +$node_b->stop; + +# pg_createsubscriber can run without --databases option +command_ok( + [ + 'pg_createsubscriber', + '--verbose', + '--pgdata' => $node_b->data_dir, + '--publisher-server' => $node_a->connstr('postgres'), + '--socketdir' => $node_b->host, + '--subscriber-port' => $node_b->port, + ], + 'run pg_createsubscriber without --databases'); + +$node_b->start; +# Verify subscriptions are created for all databases +my $subscription_count = + $node_b->safe_psql('postgres', 'SELECT COUNT (*) FROM pg_subscription'); +is($subscription_count, '4', + 'Three subscriptions are created for db3, db4, db5'); + +$node_b->stop; + # clean up $node_p->teardown_node; $node_s->teardown_node; $node_t->teardown_node; $node_f->teardown_node; +$node_a->teardown_node; +$node_b->teardown_node; done_testing(); -- 2.41.0.windows.3