Hi Zsolt, I had finally time to return to this
Please find the latest patch attached This commit fixes two critical issues : 1. Fixes dependency tracking during parallel restore. The original patch modified dependencies (adding chunks) after depCount was initialized, leading to early scheduling of dependent objects (constraints/indexes) before all data chunks were loaded. This is fixed by moving the depCount initialization loop to the end of fix_dependencies(). 2. Fixes inserts mode. The previous patch missed adding ctid range clauses to dumpTableData_insert(), causing the entire table to be dumped for each chunk when using --inserts. This is fixed by adding the segment ctid filtering logic to dumpTableData_insert() as well. Also added TAP tests to verify chunked dump/restore in insert mode. On Mon, Mar 30, 2026 at 11:32 PM Zsolt Parragi <[email protected]> wrote: > > Hello! > > A simple test causes an assertion failure in my testing, dependency > counting still doesn't seem to work correctly: > > pg_restore: >...>/pg_backup_archiver.c:5207: reduce_dependencies: > Assertion `otherte->depCount > 0' failed. > > Without assertions it results in data loss. > > 004_pg_dump_parallel also showcases the issue in my testing. > > But simple manual testing also confirms it: > > 1. create some data > > CREATE TABLE tplain (id int UNIQUE); > INSERT INTO tplain SELECT x FROM generate_series(1,1000) x; > > 2. create a dump > > dump with --max-table-segment-pages=2 > > 3. try to restore > > restore with --jobs=3
From 9c14876a83864f2e40f7a28cd55c1271a02abd24 Mon Sep 17 00:00:00 2001 From: Hannu Krosing <[email protected]> Date: Thu, 6 Aug 2026 12:15:29 +0000 Subject: [PATCH v72] Add --max-table-segment-pages option to pg_dump for parallel table dumping. This option introduces the ability to split large heap tables into segments based on a specified number of pages. These segments can then be dumped in parallel using the existing jobs infrastructure, significantly reducing the time required to dump very large tables. The implementation uses ctid-based range queries (e.g., WHERE ctid >= '(startPage,1)' AND ctid < '(endPage+1,0)') to extract specific chunks of the relation. This option only applies to tables using the "heap" table access method. This commit also fixes two critical issues: 1. Fixes dependency tracking during parallel restore. The previous patch modified dependencies (adding chunks) after depCount was initialized, leading to early scheduling of dependent objects (constraints/indexes) before all data chunks were loaded. This is fixed by moving the depCount initialization loop to the end of fix_dependencies(). 2. Fixes --inserts mode. The previous patch missed adding ctid range clauses to dumpTableData_insert(), causing the entire table to be dumped for each chunk when using --inserts. This is fixed by adding the segment ctid filtering logic to dumpTableData_insert() as well. Also added TAP tests to verify chunked dump/restore in insert mode. --- doc/src/sgml/ref/pg_dump.sgml | 24 +++ src/bin/pg_dump/pg_backup.h | 2 + src/bin/pg_dump/pg_backup_archiver.c | 119 +++++++++++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_dump.c | 189 +++++++++++++++++----- src/bin/pg_dump/pg_dump.h | 22 ++- src/bin/pg_dump/t/004_pg_dump_parallel.pl | 58 +++++++ src/fe_utils/option_utils.c | 55 +++++++ src/include/fe_utils/option_utils.h | 3 + 9 files changed, 420 insertions(+), 64 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 774be23b4f9..a1ae0150653 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -1054,6 +1054,30 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>--max-table-segment-pages=<replaceable class="parameter">npages</replaceable></option></term> + <listitem> + <para> + Dump data in segments based on number of pages in the main relation. + If the number of data pages in the relation is more than <replaceable class="parameter">npages</replaceable> + the data is split into segments based on that number of pages. + Individual segments can be dumped in parallel. + </para> + + <note> + <para> + The option <option>--max-table-segment-pages</option> is applied to only pages + in the main heap and if the table has a large TOASTed part this has to be + taken into account when deciding on the number of pages to use. + In the extreme case a single 8kB heap page can have ~200 toast pointers each + corresponding to 1GB of data. If this data is also non-compressible then a + single-page segment can dump as 200GB file. + </para> + </note> + + </listitem> + </varlistentry> + <varlistentry> <term><option>--no-comments</option></term> <listitem> diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index c7bdda1deed..1622237e804 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -27,6 +27,7 @@ #include "common/file_utils.h" #include "fe_utils/simple_list.h" #include "libpq-fe.h" +#include "storage/block.h" typedef enum trivalue @@ -179,6 +180,7 @@ typedef struct _dumpOptions bool aclsSkip; const char *lockWaitTimeout; int dump_inserts; /* 0 = COPY, otherwise rows per INSERT */ + BlockNumber max_table_segment_pages; /* chunk when relpages is above this */ /* flags for various command-line long options */ int disable_dollar_quoting; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index d7da3fc4325..899199ad1ef 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -44,6 +44,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" +#include "storage/block.h" #include "pgtar.h" #define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n" @@ -155,6 +156,7 @@ InitDumpOptions(DumpOptions *opts) opts->dumpSchema = true; opts->dumpData = true; opts->dumpStatistics = false; + opts->max_table_segment_pages = InvalidBlockNumber; } /* @@ -1979,6 +1981,25 @@ _moveBefore(TocEntry *pos, TocEntry *te) pos->prev = te; } +/* + * Add a dependency id to a DependencyList object + * This is currently used for collecting reverse + * dependencies for chunked data dump + * + * Note: duplicate dependencies are currently not eliminated + */ +void +addStandaloneDependency(DependencyList *dobj, DumpId refId) +{ + if (dobj->nDeps >= dobj->allocDeps) + { + dobj->allocDeps = (dobj->allocDeps <= 0) ? 16 : dobj->allocDeps * 2; + dobj->dependencies = pg_realloc_array(dobj->dependencies, + DumpId, dobj->allocDeps); + } + dobj->dependencies[dobj->nDeps++] = refId; +} + /* * Build index arrays for the TOC list * @@ -1998,6 +2019,7 @@ buildTocEntryArrays(ArchiveHandle *AH) AH->tocsByDumpId = pg_malloc0_array(TocEntry *, (maxDumpId + 1)); AH->tableDataId = pg_malloc0_array(DumpId, (maxDumpId + 1)); + AH->tableDataChunkIds = pg_malloc0_array(DependencyList, (maxDumpId + 1)); for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -2013,8 +2035,12 @@ buildTocEntryArrays(ArchiveHandle *AH) * TOC entry that has a DATA item. We compute this by reversing the * TABLE DATA item's dependency, knowing that a TABLE DATA item has * just one dependency and it is the TABLE item. + * + * For chunked table data, the TABLE DATA item has a description like + * "TABLE DATA (pages 100:199)", and we collect all such items as + * reverse dependencies for the parent table's entry in tableDataChunkIds. */ - if (strcmp(te->desc, "TABLE DATA") == 0 && te->nDeps > 0) + if (strncmp(te->desc, "TABLE DATA", 10) == 0 && te->nDeps > 0) { DumpId tableId = te->dependencies[0]; @@ -2026,7 +2052,14 @@ buildTocEntryArrays(ArchiveHandle *AH) if (tableId <= 0 || tableId > maxDumpId) pg_fatal("bad table dumpId for TABLE DATA item"); - AH->tableDataId[tableId] = te->dumpId; + if (te->desc[10] == '\0') /* te->desc == "TABLE DATA" */ + AH->tableDataId[tableId] = te->dumpId; + else + { + /* Chunked table data, the description is "TABLE DATA (pages %u:%u)" */ + addStandaloneDependency(&(AH->tableDataChunkIds[tableId]), te->dumpId); + pg_log_debug("Added chunked table data dependency: tableId %u + chunkId %u", + tableId, te->dumpId);} } } } @@ -2763,7 +2796,7 @@ ReadToc(ArchiveHandle *AH) strcmp(te->desc, "ACL") == 0 || strcmp(te->desc, "ACL LANGUAGE") == 0) te->section = SECTION_NONE; - else if (strcmp(te->desc, "TABLE DATA") == 0 || + else if (strncmp(te->desc, "TABLE DATA", 10) == 0 || strcmp(te->desc, "BLOBS") == 0 || strcmp(te->desc, "BLOB COMMENTS") == 0) te->section = SECTION_DATA; @@ -2993,7 +3026,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) * associated pg_shdepend rows. This is faster to restore than the * equivalent set of large object commands. */ - if (ropt->binary_upgrade && strcmp(te->desc, "TABLE DATA") == 0 && + if (ropt->binary_upgrade && strncmp(te->desc, "TABLE DATA", 10) == 0 && (te->catalogId.oid == LargeObjectMetadataRelationId || te->catalogId.oid == SharedDependRelationId)) return REQ_DATA; @@ -3237,7 +3270,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) return 0; } else if (strcmp(te->desc, "TABLE") == 0 || - strcmp(te->desc, "TABLE DATA") == 0 || + strncmp(te->desc, "TABLE DATA", 10) == 0 || strcmp(te->desc, "VIEW") == 0 || strcmp(te->desc, "FOREIGN TABLE") == 0 || strcmp(te->desc, "MATERIALIZED VIEW") == 0 || @@ -4882,19 +4915,6 @@ fix_dependencies(ArchiveHandle *AH) TocEntry *te; int i; - /* - * Initialize the depCount/revDeps/nRevDeps fields, and make sure the TOC - * items are marked as not being in any parallel-processing list. - */ - for (te = AH->toc->next; te != AH->toc; te = te->next) - { - te->depCount = te->nDeps; - te->revDeps = NULL; - te->nRevDeps = 0; - te->pending_prev = NULL; - te->pending_next = NULL; - } - /* * POST_DATA items that are shown as depending on a table need to be * re-pointed to depend on that table's data, instead. This ensures they @@ -4922,7 +4942,6 @@ fix_dependencies(ArchiveHandle *AH) te->dependencies = pg_malloc_object(DumpId); te->dependencies[0] = te2->dumpId; te->nDeps++; - te->depCount++; break; } } @@ -4931,6 +4950,22 @@ fix_dependencies(ArchiveHandle *AH) } } + /* + * Initialize the depCount/revDeps/nRevDeps fields, and make sure the TOC + * items are marked as not being in any parallel-processing list. + * + * This must be done after all dependency modifications (like repointing + * and BLOB fixes) are complete, so that depCount matches the final nDeps. + */ + for (te = AH->toc->next; te != AH->toc; te = te->next) + { + te->depCount = te->nDeps; + te->revDeps = NULL; + te->nRevDeps = 0; + te->pending_prev = NULL; + te->pending_next = NULL; + } + /* * At this point we start to build the revDeps reverse-dependency arrays, * so all changes of dependencies must be complete. @@ -5005,6 +5040,12 @@ fix_dependencies(ArchiveHandle *AH) * that parallel restore will prioritize larger jobs (index builds, FK * constraint checks, etc) over smaller ones, avoiding situations where we * end a restore with only one active job working on a large table. + * + * In case of chunked dumps, we change the depenency on table with depedency + * on the first chunk of data and add the remaining chunk ids, if any, to the + * end of depencency list + * we also calculate the fullDataLength as the sum of the lengths of chunk + * data items and use that to set the item's dataLength. */ static void repoint_table_dependencies(ArchiveHandle *AH) @@ -5020,8 +5061,9 @@ repoint_table_dependencies(ArchiveHandle *AH) for (i = 0; i < te->nDeps; i++) { olddep = te->dependencies[i]; - if (olddep <= AH->maxDumpId && - AH->tableDataId[olddep] != 0) + if (olddep > AH->maxDumpId) + continue; + if (AH->tableDataId[olddep] != 0) { DumpId tabledataid = AH->tableDataId[olddep]; TocEntry *tabledatate = AH->tocsByDumpId[tabledataid]; @@ -5031,6 +5073,39 @@ repoint_table_dependencies(ArchiveHandle *AH) pg_log_debug("transferring dependency %d -> %d to %d", te->dumpId, olddep, tabledataid); } + else if (AH->tableDataChunkIds[olddep].nDeps > 0) + { + int j; + DumpId chunkdataid; + uint64 fullDataLength; + DependencyList *deplist = &AH->tableDataChunkIds[olddep]; + + /* first in list replaces the dependency on table */ + chunkdataid = deplist->dependencies[0]; + te->dependencies[i] = chunkdataid; + fullDataLength = AH->tocsByDumpId[chunkdataid]->dataLength; + pg_log_debug("transferring chunk list %d -> %d to %d", + te->dumpId, olddep, chunkdataid); + + if (deplist->nDeps > 1) + { + /* make space */ + te->dependencies = pg_realloc_array(te->dependencies, + DumpId, + te->nDeps + deplist->nDeps - 1); + + /* the rest are appended to dependencies */ + for (j = 1; j < deplist->nDeps; j++) + { + chunkdataid = deplist->dependencies[j]; + te->dependencies[te->nDeps++] = chunkdataid; + fullDataLength += AH->tocsByDumpId[chunkdataid]->dataLength; + pg_log_debug("adding chunk list %d -> %d to %d", + te->dumpId, olddep, chunkdataid); + } + } + te->dataLength = Max(te->dataLength, fullDataLength); + } } } } @@ -5084,7 +5159,7 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te) DumpId depid = te->dependencies[i]; if (depid <= AH->maxDumpId && AH->tocsByDumpId[depid] != NULL && - ((strcmp(AH->tocsByDumpId[depid]->desc, "TABLE DATA") == 0) || + ((strncmp(AH->tocsByDumpId[depid]->desc, "TABLE DATA", 10) == 0) || strcmp(AH->tocsByDumpId[depid]->desc, "TABLE") == 0)) lockids[nlockids++] = depid; } diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index c1528d78853..92975f1003b 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -179,6 +179,13 @@ typedef enum OUTPUT_OTHERDATA, /* writing data as INSERT commands */ } ArchiverOutput; +typedef struct _DependencyList +{ + DumpId *dependencies; /* dumpIds of objects this one depends on */ + int nDeps; /* number of valid dependencies */ + int allocDeps; /* allocated size of dependencies[] */ +} DependencyList; + /* * For historical reasons, ACL items are interspersed with everything else in * a dump file's TOC; typically they're right after the object they're for. @@ -311,6 +318,7 @@ struct _archiveHandle /* arrays created after the TOC list is complete: */ struct _tocEntry **tocsByDumpId; /* TOCs indexed by dumpId */ DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ + DependencyList *tableDataChunkIds; /* dependencies indexed by dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ pg_compress_specification compression_spec; /* Requested specification for @@ -377,7 +385,7 @@ struct _tocEntry size_t defnLen; /* length of dumped definition */ /* working state while dumping/restoring */ - pgoff_t dataLength; /* item's data size; 0 if none or unknown */ + uint64 dataLength; /* item's data size; 0 if none or unknown */ int reqs; /* do we need schema and/or data of object * (REQ_* bit mask) */ bool created; /* set for DATA member if TABLE was created */ @@ -436,6 +444,8 @@ extern int TocIDRequired(ArchiveHandle *AH, DumpId id); TocEntry *getTocEntryByDumpId(ArchiveHandle *AH, DumpId id); extern bool checkSeek(FILE *fp); +extern void addStandaloneDependency(DependencyList *dobj, DumpId refId); + #define appendStringLiteralAHX(buf,str,AH) \ appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4948e6d80c7..7e546e164a5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -535,6 +535,7 @@ main(int argc, char **argv) {"exclude-extension", required_argument, NULL, 17}, {"sequence-data", no_argument, &dopt.sequence_data, 1}, {"restrict-key", required_argument, NULL, 25}, + {"max-table-segment-pages", required_argument, NULL, 26}, {NULL, 0, NULL, 0} }; @@ -799,6 +800,12 @@ main(int argc, char **argv) dopt.restrict_key = pg_strdup(optarg); break; + case 26: + if (!option_parse_uint32(optarg, "--max-table-segment-pages", 1, MaxBlockNumber, + &dopt.max_table_segment_pages)) + exit_nicely(1); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -1344,6 +1351,9 @@ help(const char *progname) printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); printf(_(" --filter=FILENAME include or exclude objects and data from dump\n" " based on expressions in FILENAME\n")); + printf(_(" --max-table-segment-pages=NUMPAGES\n" + " number of main table pages above which data is \n" + " copied out in chunks, also determines the chunk size\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --include-foreign-data=PATTERN\n" " include data of foreign tables on foreign\n" @@ -2370,7 +2380,7 @@ dumpTableData_copy(Archive *fout, const void *dcontext) * dumping an old pg_largeobject_metadata defined WITH OIDS. For other * cases a simple COPY suffices. */ - if (tdinfo->filtercond || tbinfo->relkind == RELKIND_FOREIGN_TABLE || + if (tdinfo->filtercond || is_segment(tdinfo) || tbinfo->relkind == RELKIND_FOREIGN_TABLE || (fout->dopt->binary_upgrade && fout->remoteVersion < 120000 && tbinfo->dobj.catId.oid == LargeObjectMetadataRelationId)) { @@ -2388,9 +2398,37 @@ dumpTableData_copy(Archive *fout, const void *dcontext) else appendPQExpBufferStr(q, "* "); - appendPQExpBuffer(q, "FROM %s %s) TO stdout;", + appendPQExpBuffer(q, "FROM %s %s", fmtQualifiedDumpable(tbinfo), tdinfo->filtercond ? tdinfo->filtercond : ""); + /* If it's a segment, we need to add a filter condition to select the + * right page range + * - for first segment we add "ctid < (endPage+1, 0)" + * first segment is the one with startPage == 0 + * - for last segment we add "ctid >= (startPage, 1)" + * last segment is the one with endPage == InvalidBlockNumber + * we leave to upper bound open for the case where more pages + * were added after we measured + * - for middle segments we add + * "ctid >= (startPage, 1) AND ctid < (endPage+1, 0)" + * + * "ctid < (endPage+1, 0)" instead of "ctid <= (endPage, maxtuple)" + * was chosen as range end so that we do not have to estimate the maxtuple + * + */ + if (is_segment(tdinfo)) + { + appendPQExpBufferStr(q, tdinfo->filtercond?" AND ":" WHERE "); + if(tdinfo->startPage == 0) + appendPQExpBuffer(q, "ctid < '(%u,0)'", tdinfo->endPage+1); + else if(tdinfo->endPage != InvalidBlockNumber) + appendPQExpBuffer(q, "ctid >= '(%u,1)' AND ctid < '(%u,0)'", + tdinfo->startPage, tdinfo->endPage+1); + else + appendPQExpBuffer(q, "ctid >= '(%u,1)'", tdinfo->startPage); + } + + appendPQExpBuffer(q, ") TO stdout;"); } else { @@ -2398,6 +2436,10 @@ dumpTableData_copy(Archive *fout, const void *dcontext) fmtQualifiedDumpable(tbinfo), column_list); } + + if (is_segment(tdinfo)) + pg_log_debug("CHUNKING: data query: %s", q->data); + res = ExecuteSqlQuery(fout, q->data, PGRES_COPY_OUT); PQclear(res); destroyPQExpBuffer(clistBuf); @@ -2557,6 +2599,18 @@ dumpTableData_insert(Archive *fout, const void *dcontext) if (tdinfo->filtercond) appendPQExpBuffer(q, " %s", tdinfo->filtercond); + if (is_segment(tdinfo)) + { + appendPQExpBufferStr(q, tdinfo->filtercond ? " AND " : " WHERE "); + if (tdinfo->startPage == 0) + appendPQExpBuffer(q, "ctid < '(%u,0)'", tdinfo->endPage + 1); + else if (tdinfo->endPage != InvalidBlockNumber) + appendPQExpBuffer(q, "ctid >= '(%u,1)' AND ctid < '(%u,0)'", + tdinfo->startPage, tdinfo->endPage + 1); + else + appendPQExpBuffer(q, "ctid >= '(%u,1)'", tdinfo->startPage); + } + ExecuteSqlStatement(fout, q->data); while (1) @@ -2893,42 +2947,89 @@ dumpTableData(Archive *fout, const TableDataInfo *tdinfo) { TocEntry *te; - te = ArchiveEntry(fout, tdinfo->dobj.catId, tdinfo->dobj.dumpId, - ARCHIVE_OPTS(.tag = tbinfo->dobj.name, - .namespace = tbinfo->dobj.namespace->dobj.name, - .owner = tbinfo->rolname, - .description = "TABLE DATA", - .section = SECTION_DATA, - .createStmt = tdDefn, - .copyStmt = copyStmt, - .deps = &(tbinfo->dobj.dumpId), - .nDeps = 1, - .dumpFn = dumpFn, - .dumpArg = tdinfo)); - - /* - * Set the TocEntry's dataLength in case we are doing a parallel dump - * and want to order dump jobs by table size. We choose to measure - * dataLength in table pages (including TOAST pages) during dump, so - * no scaling is needed. - * - * However, relpages is declared as "integer" in pg_class, and hence - * also in TableInfo, but it's really BlockNumber a/k/a unsigned int. - * Cast so that we get the right interpretation of table sizes - * exceeding INT_MAX pages. + /* data chunking works off relpages, which are computed exactly using + * pg_relation_size() when --max-table-segment-pages was set + * + * We also don't chunk if table access method is not "heap" + * TODO: we may add chunking for other access methods later, maybe + * based on primary key tranges */ - te->dataLength = (BlockNumber) tbinfo->relpages; - te->dataLength += (BlockNumber) tbinfo->toastpages; + if (tbinfo->relpages <= dopt->max_table_segment_pages || + strcmp(tbinfo->amname, "heap") != 0) + { + te = ArchiveEntry(fout, tdinfo->dobj.catId, tdinfo->dobj.dumpId, + ARCHIVE_OPTS(.tag = tbinfo->dobj.name, + .namespace = tbinfo->dobj.namespace->dobj.name, + .owner = tbinfo->rolname, + .description = "TABLE DATA", + .section = SECTION_DATA, + .createStmt = tdDefn, + .copyStmt = copyStmt, + .deps = &(tbinfo->dobj.dumpId), + .nDeps = 1, + .dumpFn = dumpFn, + .dumpArg = tdinfo)); - /* - * If pgoff_t is only 32 bits wide, the above refinement is useless, - * and instead we'd better worry about integer overflow. Clamp to - * INT_MAX if the correct result exceeds that. - */ - if (sizeof(te->dataLength) == 4 && - (tbinfo->relpages < 0 || tbinfo->toastpages < 0 || - te->dataLength < 0)) - te->dataLength = INT_MAX; + /* + * Set the TocEntry's dataLength in case we are doing a parallel dump + * and want to order dump jobs by table size. We choose to measure + * dataLength in table pages (including TOAST pages) during dump, so + * no scaling is needed. + * + * While pg_class.relpages which stores BlockNumber, a/k/a unsigned int, + * is declared as "integer" we convert it back and store it as + * BlockNumber in TableInfo. + * And dataLenght is pgoff_t (long int) so does now overflow for + * 2 x UINT32_MAX + */ + te->dataLength = tbinfo->relpages; + te->dataLength += tbinfo->toastpages; + } + else + { + uint64 current_chunk_start = 0; + PQExpBuffer chunk_desc = createPQExpBuffer(); + + while (current_chunk_start < tbinfo->relpages) + { + TableDataInfo *chunk_tdinfo = (TableDataInfo *) pg_malloc(sizeof(TableDataInfo)); + + memcpy(chunk_tdinfo, tdinfo, sizeof(TableDataInfo)); + AssignDumpId(&chunk_tdinfo->dobj); + addObjectDependency(&chunk_tdinfo->dobj, tbinfo->dobj.dumpId); + chunk_tdinfo->startPage = (BlockNumber) current_chunk_start; + chunk_tdinfo->endPage = chunk_tdinfo->startPage + dopt->max_table_segment_pages - 1; + + current_chunk_start += dopt->max_table_segment_pages; + if (current_chunk_start >= tbinfo->relpages) + chunk_tdinfo->endPage = InvalidBlockNumber; /* last chunk is for "all the rest" */ + + printfPQExpBuffer(chunk_desc, "TABLE DATA (pages %u:%u)", chunk_tdinfo->startPage, chunk_tdinfo->endPage); + + te = ArchiveEntry(fout, chunk_tdinfo->dobj.catId, chunk_tdinfo->dobj.dumpId, + ARCHIVE_OPTS(.tag = tbinfo->dobj.name, + .namespace = tbinfo->dobj.namespace->dobj.name, + .owner = tbinfo->rolname, + .description = chunk_desc->data, + .section = SECTION_DATA, + .createStmt = tdDefn, + .copyStmt = copyStmt, + .deps = &(tbinfo->dobj.dumpId), + .nDeps = 1, + .dumpFn = dumpFn, + .dumpArg = chunk_tdinfo)); + + if(chunk_tdinfo->endPage == InvalidBlockNumber) + te->dataLength = tbinfo->relpages - chunk_tdinfo->startPage; + else + te->dataLength = dopt->max_table_segment_pages; + /* let's assume toast pages distribute evenly among chunks */ + if(tbinfo->relpages) + te->dataLength += te->dataLength * tbinfo->toastpages / tbinfo->relpages; + } + + destroyPQExpBuffer(chunk_desc); + } } destroyPQExpBuffer(copyBuf); @@ -3055,6 +3156,8 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo) tdinfo->dobj.namespace = tbinfo->dobj.namespace; tdinfo->tdtable = tbinfo; tdinfo->filtercond = NULL; /* might get set later */ + tdinfo->startPage = InvalidBlockNumber; /* we use this as indication that no chunking is needed */ + tdinfo->endPage = InvalidBlockNumber; addObjectDependency(&tdinfo->dobj, tbinfo->dobj.dumpId); /* A TableDataInfo contains data, of course */ @@ -7219,8 +7322,16 @@ getTables(Archive *fout, int *numTables) "c.relnamespace, c.relkind, c.reltype, " "c.relowner, " "c.relchecks, " - "c.relhasindex, c.relhasrules, c.relpages, " - "c.reltuples, c.relallvisible, "); + "c.relhasindex, c.relhasrules, "); + + /* fetch current relation size if chunking is requested */ + if(dopt->max_table_segment_pages != InvalidBlockNumber) + appendPQExpBufferStr(query, "pg_relation_size(c.oid)/current_setting('block_size')::int AS relpages, "); + else + /* pg_class.relpages stores BlockNumber (uint32) in an int field, convert to oid to get unsigned int out */ + appendPQExpBufferStr(query, "c.relpages::oid, "); + + appendPQExpBufferStr(query, "c.reltuples, c.relallvisible, "); if (fout->remoteVersion >= 180000) appendPQExpBufferStr(query, "c.relallfrozen, "); @@ -7436,7 +7547,7 @@ getTables(Archive *fout, int *numTables) tblinfo[i].ncheck = atoi(PQgetvalue(res, i, i_relchecks)); tblinfo[i].hasindex = (strcmp(PQgetvalue(res, i, i_relhasindex), "t") == 0); tblinfo[i].hasrules = (strcmp(PQgetvalue(res, i, i_relhasrules), "t") == 0); - tblinfo[i].relpages = atoi(PQgetvalue(res, i, i_relpages)); + tblinfo[i].relpages = strtoul(PQgetvalue(res, i, i_relpages), NULL, 10); if (PQgetisnull(res, i, i_toastpages)) tblinfo[i].toastpages = 0; else diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 5a6726d8b12..84e682d585f 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -16,6 +16,7 @@ #include "pg_backup.h" #include "catalog/pg_publication_d.h" +#include "storage/block.h" #define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ? 1 : 0) ) @@ -335,7 +336,11 @@ typedef struct _tableInfo Oid owning_tab; /* OID of table owning sequence */ int owning_col; /* attr # of column owning sequence */ bool is_identity_sequence; - int32 relpages; /* table's size in pages (from pg_class) */ + BlockNumber relpages; /* table's size in pages (from pg_class) + * converted to unsigned integer + * when --max-table-segment-pages is set + * the computed from pg_relation_size() + */ int toastpages; /* toast table's size in pages, if any */ bool interesting; /* true if need to collect more data */ @@ -413,8 +418,21 @@ typedef struct _tableDataInfo DumpableObject dobj; TableInfo *tdtable; /* link to table to dump */ char *filtercond; /* WHERE condition to limit rows dumped */ + /* startPage and endPage to support segmented dump */ + BlockNumber startPage; /* As we always know the lowest segment page + * number we can use InvalidBlockNumber here + * to recognize no segmenting case. + * When 0 for the first page of first + * segment we can omit in range query */ + BlockNumber endPage; /* last page in segment for page-range dump, + * startPage+max_table_segment_pages-1 for + * most segments, but InvalidBlockNumber for + * the last one to indicate open range + */ } TableDataInfo; +#define is_segment(tdiptr) ((tdiptr)->startPage != InvalidBlockNumber) + typedef struct _indxInfo { DumpableObject dobj; @@ -449,7 +467,7 @@ typedef struct _relStatsInfo { DumpableObject dobj; Oid relid; - int32 relpages; + BlockNumber relpages; char *reltuples; int32 relallvisible; int32 relallfrozen; diff --git a/src/bin/pg_dump/t/004_pg_dump_parallel.pl b/src/bin/pg_dump/t/004_pg_dump_parallel.pl index 738f34b1c1b..9dbbb60725b 100644 --- a/src/bin/pg_dump/t/004_pg_dump_parallel.pl +++ b/src/bin/pg_dump/t/004_pg_dump_parallel.pl @@ -11,6 +11,8 @@ use Test::More; my $dbname1 = 'regression_src'; my $dbname2 = 'regression_dest1'; my $dbname3 = 'regression_dest2'; +my $dbname4 = 'regression_dest3'; +my $dbname5 = 'regression_dest4'; my $node = PostgreSQL::Test::Cluster->new('main'); $node->init; @@ -21,6 +23,8 @@ my $backupdir = $node->backup_dir; $node->run_log([ 'createdb', $dbname1 ]); $node->run_log([ 'createdb', $dbname2 ]); $node->run_log([ 'createdb', $dbname3 ]); +$node->run_log([ 'createdb', $dbname4 ]); +$node->run_log([ 'createdb', $dbname5 ]); $node->safe_psql( $dbname1, @@ -87,4 +91,58 @@ $node->command_ok( ], 'parallel restore as inserts'); +$node->command_ok( + [ + 'pg_dump', + '--format' => 'directory', + '--max-table-segment-pages' => 2, + '--no-sync', + '--jobs' => 2, + '--file' => "$backupdir/dump3", + $node->connstr($dbname1), + ], + 'parallel dump with chunks of two heap pages'); + +$node->command_ok( + [ + 'pg_restore', '--verbose', + '--dbname' => $node->connstr($dbname4), + '--jobs' => 3, + "$backupdir/dump3", + ], + 'parallel restore with chunks of two heap pages'); + +my $table = 'tplain'; +my $tablehash_query = "SELECT '$table', sum(hashtext(t::text)), count(*) FROM $table AS t"; + +my $result_1 = $node->safe_psql($dbname1, $tablehash_query); +my $result_4 = $node->safe_psql($dbname4, $tablehash_query); + +is($result_4, $result_1, "Hash check for $table: restored db ($result_4) vs original db ($result_1)"); + +$node->command_ok( + [ + 'pg_dump', + '--format' => 'directory', + '--max-table-segment-pages' => 2, + '--inserts', + '--no-sync', + '--jobs' => 2, + '--file' => "$backupdir/dump4", + $node->connstr($dbname1), + ], + 'parallel dump with chunks of two heap pages using inserts'); + +$node->command_ok( + [ + 'pg_restore', '--verbose', + '--dbname' => $node->connstr($dbname5), + '--jobs' => 3, + "$backupdir/dump4", + ], + 'parallel restore with chunks of two heap pages using inserts'); + +my $result_5 = $node->safe_psql($dbname5, $tablehash_query); +is($result_5, $result_1, "Hash check for $table (inserts): restored db ($result_5) vs original db ($result_1)"); + done_testing(); diff --git a/src/fe_utils/option_utils.c b/src/fe_utils/option_utils.c index c8bba075c1f..c2e12907a62 100644 --- a/src/fe_utils/option_utils.c +++ b/src/fe_utils/option_utils.c @@ -83,6 +83,61 @@ option_parse_int(const char *optarg, const char *optname, return true; } +/* + * option_parse_uint32 + * + * Parse unsigned integer value for an option. If the parsing is successful, + * returns true and stores the result in *result if that's given; + * if parsing fails, returns false. + */ +bool +option_parse_uint32(const char *optarg, const char *optname, + uint32 min_range, uint32 max_range, + uint32 *result) +{ + char *endptr; + unsigned long val; + + /* Fail if there is a minus sign at the start of value */ + while(isspace((unsigned char) *optarg)) + optarg++; + if(*optarg == '-') + { + pg_log_error("value \"%s\" for option %s can not be negative", + optarg, optname); + return false; + } + + errno = 0; + val = strtoul(optarg, &endptr, 10); + + /* + * Skip any trailing whitespace; if anything but whitespace remains before + * the terminating character, fail. + */ + while (*endptr != '\0' && isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr != '\0') + { + pg_log_error("invalid value \"%s\" for option %s", + optarg, optname); + return false; + } + + /* as min_range and max_range are uint32 then the range check will + * catch the case where unsigned long val is outside 32 bit range */ + if (errno == ERANGE || val < min_range || val > max_range) + { + pg_log_error("%s not in range %u..%u", optname, min_range, max_range); + return false; + } + + if (result) + *result = (uint32) val; + return true; +} + /* * Provide strictly harmonized handling of the --sync-method option. */ diff --git a/src/include/fe_utils/option_utils.h b/src/include/fe_utils/option_utils.h index f70b6159099..b04f1fb0455 100644 --- a/src/include/fe_utils/option_utils.h +++ b/src/include/fe_utils/option_utils.h @@ -22,6 +22,9 @@ extern void handle_help_version_opts(int argc, char *argv[], extern bool option_parse_int(const char *optarg, const char *optname, int min_range, int max_range, int *result); +extern bool option_parse_uint32(const char *optarg, const char *optname, + uint32 min_range, uint32 max_range, + uint32 *result); extern bool parse_sync_method(const char *optarg, DataDirSyncMethod *sync_method); extern void check_mut_excl_opts_internal(int n, ...); -- 2.55.0.629.g250fe7f194-goog
