This is an automated email from the ASF dual-hosted git repository. maxyang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit e20a48899d2fad309dcf416e0a7be17d46bd954d Author: liushengsong <[email protected]> AuthorDate: Fri Nov 28 17:54:50 2025 +0800 Fix: fix pg_upgrade and pg_dump for upgrade 1. Add database type and version to pg_upgrade and pg_dump to distinguish different catalogs. 2. Fixed serveral issues encountered when using pg_dump and pg_upgrade. --- src/backend/utils/adt/pg_upgrade_support.c | 14 ++++++ src/bin/pg_dump/pg_backup.h | 14 ++++++ src/bin/pg_dump/pg_backup_db.c | 27 +++++++++++ src/bin/pg_dump/pg_dump.c | 13 +++--- src/bin/pg_dump/pg_dumpall.c | 6 +++ src/bin/pg_upgrade/check.c | 4 +- src/bin/pg_upgrade/controldata.c | 2 +- src/bin/pg_upgrade/exec.c | 7 +-- src/bin/pg_upgrade/greenplum/check_gp.c | 8 ++++ src/bin/pg_upgrade/info.c | 8 +++- src/bin/pg_upgrade/pg_upgrade.c | 72 ++++++++++++++++++++++++++++++ src/bin/pg_upgrade/pg_upgrade.h | 14 ++++++ src/bin/pg_upgrade/server.c | 2 +- src/bin/pg_upgrade/tablespace.c | 5 ++- src/bin/pg_upgrade/version.c | 8 +++- src/include/catalog/pg_proc.dat | 4 ++ 16 files changed, 192 insertions(+), 16 deletions(-) diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c index 38347caa516..5d99e1346f3 100644 --- a/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/backend/utils/adt/pg_upgrade_support.c @@ -20,6 +20,7 @@ #include "catalog/pg_class.h" #include "catalog/pg_enum.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_tablespace.h" #include "catalog/pg_type.h" #include "cdb/cdbvars.h" #include "commands/extension.h" @@ -38,6 +39,19 @@ do { \ errmsg("function can only be called when server is in binary upgrade mode"))); \ } while (0) +Datum +binary_upgrade_set_next_pg_tablespace_oid(PG_FUNCTION_ARGS) +{ + Oid tablespaceoid = PG_GETARG_OID(0); + char *tablespacename = GET_STR(PG_GETARG_TEXT_P(1)); + + CHECK_IS_BINARY_UPGRADE; + AddPreassignedOidFromBinaryUpgrade(tablespaceoid, TableSpaceRelationId, tablespacename, + InvalidOid, InvalidOid, InvalidOid); + + PG_RETURN_VOID(); +} + Datum binary_upgrade_set_next_pg_type_oid(PG_FUNCTION_ARGS) { diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index df7470feb2d..e7f8140fb24 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -184,6 +184,18 @@ typedef struct _dumpOptions int do_nothing; } DumpOptions; + +typedef enum +{ + Greenplum, + Cloudberry +} DatabaseType; + +typedef struct +{ + DatabaseType type; + int version; +} DatabaseVersion; /* * We may want to have some more user-readable data, but in the mean * time this gives us some abstraction and type checking. @@ -216,6 +228,8 @@ typedef struct Archive bool exit_on_error; /* whether to exit on SQL errors... */ int n_errors; /* number of errors (if no die) */ + DatabaseVersion version; + /* The rest is private */ } Archive; diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index e59cf998dec..967e35ec7f2 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -35,6 +35,10 @@ _check_database_version(ArchiveHandle *AH) const char *remoteversion_str; int remoteversion; PGresult *res; + char *version; + char dbstring[1024]; + char dbstring2[1024]; + int v1; remoteversion_str = PQparameterStatus(AH->connection, "server_version"); remoteversion = PQserverVersion(AH->connection); @@ -68,6 +72,29 @@ _check_database_version(ArchiveHandle *AH) } else AH->public.isStandby = false; + + res = ExecuteSqlQueryForSingleRow((Archive *) AH, "SELECT version()"); + + version = PQgetvalue(res, 0, 0); + + if(sscanf(version, "%*[^(](%s %s %d.%*d.%*d-%*[^)])", dbstring, dbstring2, &v1) != 3) + pg_log_error("could not get version output from select version();\n"); + + if (strcasecmp("Greenplum", dbstring) == 0 && strcasecmp("Database", dbstring2) == 0) + { + AH->public.version.type = Greenplum; + AH->public.version.version = v1; + + } else if((strcasecmp("Cloudberry", dbstring) == 0 && strcasecmp("Database", dbstring2) == 0) + || (strcasecmp("Apache", dbstring) == 0 && strcasecmp("Cloudberry", dbstring2) == 0)) + { + AH->public.version.type = Cloudberry; + AH->public.version.version = v1; + } + else + pg_log_error("could not upgrade from non Greenplum/Cloudberry version: %s %s\n", dbstring, dbstring2); + + PQclear(res); } /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2293bc6d41a..7ea694ce07d 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -7384,7 +7384,7 @@ getTables(Archive *fout, int *numTables) "%s AS ispartition, " "%s AS partbound, " "c.relisivm AS isivm, " - "c.relisdynamic AS isdynamic " + "%s" "FROM pg_class c " "LEFT JOIN pg_depend d ON " "(c.relkind = '%c' AND " @@ -7414,6 +7414,7 @@ getTables(Archive *fout, int *numTables) partkeydef, ispartition, partbound, + (fout->version.type == Cloudberry && fout->version.version >= 2) ? "c.relisdynamic AS isdynamic " : "false AS isdynamic ", RELKIND_SEQUENCE, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, @@ -15596,18 +15597,18 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) if (fout->remoteVersion >= 110000) appendPQExpBufferStr(query, "aggfinalmodify,\n" - "aggmfinalmodify\n"); + "aggmfinalmodify,\n"); else appendPQExpBufferStr(query, "'0' AS aggfinalmodify,\n" - "'0' AS aggmfinalmodify\n"); + "'0' AS aggmfinalmodify,\n"); - if (fout->remoteVersion >= 140000) + if (fout->remoteVersion >= 140000 && fout->version.type == Cloudberry && fout->version.version >= 2) appendPQExpBufferStr(query, - "aggrepsafeexec,\n"); + "aggrepsafeexec\n"); else appendPQExpBufferStr(query, - "false AS aggrepsafeexec,\n"); + "false AS aggrepsafeexec\n"); appendPQExpBuffer(query, "FROM pg_catalog.pg_aggregate a, pg_catalog.pg_proc p " diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 73310951b54..b193448d2a4 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1759,6 +1759,12 @@ dumpTablespaces(PGconn *conn) /* needed for buildACLCommands() */ fspcname = pg_strdup(fmtId(spcname)); + if (binary_upgrade) + { + appendPQExpBufferStr(buf, "\n-- For binary upgrade, must preserve pg_tablespace oid\n"); + appendPQExpBuffer(buf, "SELECT pg_catalog.binary_upgrade_set_next_pg_tablespace_oid('%u'::pg_catalog.oid, '%s'::text);\n", spcoid, spcname); + } + appendPQExpBuffer(buf, "CREATE TABLESPACE %s", spcname); appendPQExpBuffer(buf, " OWNER %s", fmtId(spcowner)); diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 1cf10dace16..d0905f3d588 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -576,7 +576,7 @@ check_for_new_tablespace_dir(ClusterInfo *new_cluster) os_info.old_tablespaces[tblnum], new_cluster->tablespace_suffix); - if (stat(new_tablespace_dir, &statbuf) == 0 || errno != ENOENT) + if (is_greenplum_dispatcher_mode() && (stat(new_tablespace_dir, &statbuf) == 0 || errno != ENOENT)) gp_fatal_log("new cluster tablespace directory already exists: \"%s\"\n", new_tablespace_dir); } @@ -1598,6 +1598,8 @@ check_for_cluster_key_failure(ClusterInfo *cluster) { struct stat buffer; + prep_status("Checking for presence of pg_cryptokeys"); + if (stat (KMGR_DIR_PID, &buffer) == 0) { if (cluster == &old_cluster) diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 78749e9d310..2b300819678 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -555,7 +555,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.data_checksum_version = str2uint(p); got_data_checksum_version = true; } - else if ((p = strstr(bufin, "Cluster file encryption method:")) != NULL) + else if ((p = strstr(bufin, "File encryption method:")) != NULL) { p = strchr(p, ':'); diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c index 09faaf8d096..81b1aef91c4 100644 --- a/src/bin/pg_upgrade/exec.c +++ b/src/bin/pg_upgrade/exec.c @@ -52,11 +52,12 @@ get_bin_version(ClusterInfo *cluster) pclose(output); - if (sscanf(cmd_output, "%*s (%s %s) %d.%d", dbstring, dbstring2, &v1, &v2) < 1) + if (sscanf(cmd_output, "%*s (%s %[^)]) %d.%d", dbstring, dbstring2, &v1, &v2) != 4) pg_fatal("could not get pg_ctl version output from %s\n", cmd); - if ((strcmp("Greenplum", dbstring) == 0 && strcmp("Database", dbstring2) == 0) \ - || (strcmp("Apache", dbstring2) == 0 && strcmp("Cloudberry", dbstring2) == 0)) + if (!((strcmp("Greenplum", dbstring) == 0 && strcmp("Database", dbstring2) == 0) \ + || (strcmp("Cloudberry", dbstring) == 0 && strcmp("Database", dbstring2) == 0) + || (strcmp("Apache", dbstring) == 0 && strcmp("Cloudberry", dbstring2) == 0))) pg_fatal("could not upgrade from non Greenplum/Cloudberry version: %s %s\n", dbstring, dbstring2); if (v1 < 10) diff --git a/src/bin/pg_upgrade/greenplum/check_gp.c b/src/bin/pg_upgrade/greenplum/check_gp.c index 7b95841c097..d9893fab887 100644 --- a/src/bin/pg_upgrade/greenplum/check_gp.c +++ b/src/bin/pg_upgrade/greenplum/check_gp.c @@ -665,6 +665,14 @@ check_for_plpython2_dependent_functions(ClusterInfo *cluster) char output_path[MAXPGPATH]; bool found = false; + + if (cluster->version.type == Cloudberry) + { + prep_status("Skip checking for plpython2 functions, CBDB use plpython3"); + check_ok(); + return ; + } + prep_status("Checking for functions dependent on plpython2"); snprintf(output_path, sizeof(output_path), "%s/%s", diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 15372af5991..da2f3cb9e38 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -119,9 +119,15 @@ gen_db_file_maps(DbInfo *old_db, DbInfo *new_db, * Verify that rels of same OID have same name. The namespace name * should always match, but the relname might not match for TOAST * tables (and, therefore, their indexes). + * + * XXX GPDB: for TOAST tables, don't insist on a match at all + * yet; there are other ways for us to get mismatched names. Ideally + * this will go away eventually. */ if (strcmp(old_rel->nspname, new_rel->nspname) != 0 || - strcmp(old_rel->relname, new_rel->relname) != 0) + (strcmp(old_rel->relname, new_rel->relname) != 0 && + (/* GET_MAJOR_VERSION(old_cluster.major_version) >= 900 || */ + strcmp(old_rel->nspname, "pg_toast") != 0))) { pg_log(PG_WARNING, "Relation names for OID %u in database \"%s\" do not match: " "old name \"%s.%s\", new name \"%s.%s\"\n", diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index eee38da194f..054d1b07c29 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -59,6 +59,7 @@ static void copy_xact_xlog_xid(void); static void set_frozenxids(bool minmxid_only); static void make_outputdirs(char *pgdata); static void setup(char *argv0, bool *live_check); +static void get_cluster_version(ClusterInfo *cluster); static void copy_subdir_files(const char *old_subdir, const char *new_subdir); @@ -170,6 +171,9 @@ main(int argc, char **argv) umask(pg_mode_mask); } + get_cluster_version(&old_cluster); + get_cluster_version(&new_cluster); + check_and_dump_old_cluster(live_check, &sequence_script_file_name); @@ -304,6 +308,74 @@ main(int argc, char **argv) return 0; } + +void +get_cluster_version(ClusterInfo *cluster) +{ + PGconn *conn; + PGresult *res; + char query[QUERY_ALLOC]; + char *version; + int i_version; + int ntups; + int dbid; + int i_dbid; + char dbstring[MAX_STRING]; + char dbstring2[MAX_STRING]; + int v1; + + start_postmaster(cluster, false); + + conn = connectToServer(cluster, "template1"); + + snprintf(query, sizeof(query), "SELECT version()"); + + res = executeQueryOrDie(conn, "%s", query); + + i_version = PQfnumber(res, "version"); + + ntups = PQntuples(res); + Assert(ntups == 1); + + version = PQgetvalue(res, 0, i_version); + + if(sscanf(version, "%*[^(](%s %s %d.%*d.%*d-%*[^)])", dbstring, dbstring2, &v1) != 3) + pg_fatal("could not get version output from select version();\n"); + + if (strcasecmp("Greenplum", dbstring) == 0 && strcasecmp("Database", dbstring2) == 0) + { + cluster->version.type = Greenplum; + cluster->version.version = v1; + + } else if((strcasecmp("Cloudberry", dbstring) == 0 && strcasecmp("Database", dbstring2) == 0) + || (strcasecmp("Apache", dbstring) == 0 && strcasecmp("Cloudberry", dbstring2) == 0)) + { + cluster->version.type = Cloudberry; + cluster->version.version = v1; + } + else + pg_fatal("could not upgrade from non Greenplum/Cloudberry version: %s %s\n", dbstring, dbstring2); + + res = executeQueryOrDie(conn, "%s", "show gp_dbid;"); + + i_dbid = PQfnumber(res, "gp_dbid"); + + ntups = PQntuples(res); + Assert(ntups == 1); + + dbid = atoi(PQgetvalue(res, 0, i_dbid)); + + cluster->dbid = dbid; + + PQclear(res); + + PQfinish(conn); + + stop_postmaster(cluster); + + return; +} + #ifdef WIN32 typedef BOOL(WINAPI * __CreateRestrictedToken) (HANDLE, DWORD, DWORD, PSID_AND_ATTRIBUTES, DWORD, PLUID_AND_ATTRIBUTES, DWORD, PSID_AND_ATTRIBUTES, PHANDLE); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 1ab50c4f830..9b1775e5435 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -351,6 +351,18 @@ typedef enum typedef long pgpid_t; +typedef enum +{ + Greenplum, + Cloudberry +} DatabaseType; + +typedef struct +{ + DatabaseType type; + int version; +} DatabaseVersion; + /* * cluster @@ -373,6 +385,8 @@ typedef struct char major_version_str[64]; /* string PG_VERSION of cluster */ uint32 bin_version; /* version returned from pg_ctl */ const char *tablespace_suffix; /* directory specification */ + DatabaseVersion version; + int32 dbid; } ClusterInfo; diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c index c2ab2872630..20e12bc455f 100644 --- a/src/bin/pg_upgrade/server.c +++ b/src/bin/pg_upgrade/server.c @@ -260,7 +260,7 @@ start_postmaster(ClusterInfo *cluster, bool report_and_exit_on_error) } snprintf(cmd, sizeof(cmd), - "\"%s/pg_ctl\" -w -l \"%s/%s\" -D \"%s\" -o \"-p %d -c %s -b%s %s %s%s\" start", + "unset LD_LIBRARY_PATH;\"%s/pg_ctl\" -w -l \"%s/%s\" -D \"%s\" -o \"-p %d -c %s -b%s %s %s%s\" start", cluster->bindir, log_opts.logdir, SERVER_LOG_FILE, diff --git a/src/bin/pg_upgrade/tablespace.c b/src/bin/pg_upgrade/tablespace.c index 741a856b6c7..98f4041ecbf 100644 --- a/src/bin/pg_upgrade/tablespace.c +++ b/src/bin/pg_upgrade/tablespace.c @@ -110,7 +110,8 @@ set_tablespace_directory_suffix(ClusterInfo *cluster) /* This cluster has a version-specific subdirectory */ /* The leading slash is needed to start a new directory. */ - cluster->tablespace_suffix = psprintf("/PG_%s_%d", - cluster->major_version_str, + cluster->tablespace_suffix = psprintf("/%d/GPDB_%d_%d", + cluster->dbid, + cluster->version.version, cluster->controldata.cat_ver); } diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 7acee2b563e..02947234ca0 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -112,6 +112,12 @@ check_for_data_types_usage(ClusterInfo *cluster, FILE *script = NULL; int dbnum; + if (cluster->version.type == Cloudberry && cluster->version.version <= 2) + { + pg_log(PG_REPORT, "Skip checking for data type, current version CBDB do not support recursive query"); + return false; + } + for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) { PQExpBuffer cte_oids; @@ -547,7 +553,7 @@ report_extension_updates(ClusterInfo *cluster) if (found) { report_status(PG_REPORT, "notice"); - gp_fatal_log( + pg_log(PG_REPORT, "| Your installation contains extensions that should be updated\n" "| with the ALTER EXTENSION command. The file\n" "| %s\n" diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2a0d5adef40..a47b1ef1615 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11055,6 +11055,10 @@ proname => 'binary_upgrade_set_next_array_pg_type_oid', provolatile => 'v', proparallel => 'r', prorettype => 'void', proargtypes => 'oid oid text', prosrc => 'binary_upgrade_set_next_array_pg_type_oid' }, +{ oid => '4545', descr => 'for use by pg_upgrade', + proname => 'binary_upgrade_set_next_pg_tablespace_oid', provolatile => 'v', + proparallel => 'r', prorettype => 'void', proargtypes => 'oid text', + prosrc => 'binary_upgrade_set_next_pg_tablespace_oid' }, { oid => '4390', descr => 'for use by pg_upgrade', proname => 'binary_upgrade_set_next_multirange_pg_type_oid', provolatile => 'v', proparallel => 'r', prorettype => 'void', --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
