On 04.02.2019 10:04, Michael Paquier wrote: > On Tue, Dec 18, 2018 at 10:41:48AM +0500, Andrey Lepikhov wrote: >> Ok. It is used only for demonstration. > > The latest patch set needs a rebase, so moved to next CF, waiting on > author as this got no reviews. The new version in attachment. > -- > Michael >
-- Andrey Lepikhov Postgres Professional https://postgrespro.com The Russian Postgres Company
From afc270f37cd082fb6e9f4ad694ea4cb123d98062 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" <a.lepik...@postgrespro.ru> Date: Tue, 5 Feb 2019 08:11:47 +0500 Subject: [PATCH 1/4] Relation-into-WAL-function --- src/backend/access/spgist/spgutils.c | 1 + src/backend/access/transam/generic_xlog.c | 39 +++++++++++++++++++++++ src/include/access/generic_xlog.h | 3 ++ 3 files changed, 43 insertions(+) diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index 8e63c1fad2..b782bc2338 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -550,6 +550,7 @@ SpGistInitBuffer(Buffer b, uint16 f) { Assert(BufferGetPageSize(b) == BLCKSZ); SpGistInitPage(BufferGetPage(b), f); + MarkBufferDirty(b); } /* diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c index 5b00b7275b..643da48345 100644 --- a/src/backend/access/transam/generic_xlog.c +++ b/src/backend/access/transam/generic_xlog.c @@ -542,3 +542,42 @@ generic_mask(char *page, BlockNumber blkno) mask_unused_space(page); } + +/* + * Function to write generic xlog for every existing block of a relation. + * Caller is responsible for locking the relation exclusively. + */ +void +generic_log_relation(Relation rel) +{ + BlockNumber blkno; + BlockNumber nblocks = RelationGetNumberOfBlocks(rel); + + for (blkno = 0; blkno < nblocks; ) + { + GenericXLogState *state; + Buffer buffer[MAX_GENERIC_XLOG_PAGES]; + int i, + blocks_pack; + + CHECK_FOR_INTERRUPTS(); + + blocks_pack = ((nblocks-blkno) < MAX_GENERIC_XLOG_PAGES) ? + (nblocks-blkno) : MAX_GENERIC_XLOG_PAGES; + + state = GenericXLogStart(rel); + + for (i = 0 ; i < blocks_pack; i++) + { + buffer[i] = ReadBuffer(rel, blkno); + LockBuffer(buffer[i], BUFFER_LOCK_EXCLUSIVE); + GenericXLogRegisterBuffer(state, buffer[i], GENERIC_XLOG_FULL_IMAGE); + blkno++; + } + + GenericXLogFinish(state); + + for (i = 0 ; i < blocks_pack; i++) + UnlockReleaseBuffer(buffer[i]); + } +} diff --git a/src/include/access/generic_xlog.h b/src/include/access/generic_xlog.h index cb5b5b713a..e3bbf014cc 100644 --- a/src/include/access/generic_xlog.h +++ b/src/include/access/generic_xlog.h @@ -42,4 +42,7 @@ extern const char *generic_identify(uint8 info); extern void generic_desc(StringInfo buf, XLogReaderState *record); extern void generic_mask(char *pagedata, BlockNumber blkno); +/* other utils */ +extern void generic_log_relation(Relation rel); + #endif /* GENERIC_XLOG_H */ -- 2.17.1
From 25dc935da9d5502a06fb9dc1eea4912fa0f48be1 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" <a.lepik...@postgrespro.ru> Date: Tue, 5 Feb 2019 08:13:29 +0500 Subject: [PATCH 2/4] GIN-Optimal-WAL-Usage --- src/backend/access/gin/ginbtree.c | 6 ++--- src/backend/access/gin/gindatapage.c | 10 ++++---- src/backend/access/gin/ginentrypage.c | 2 +- src/backend/access/gin/gininsert.c | 30 ++++++++++-------------- src/backend/access/gin/ginutil.c | 4 ++-- src/backend/access/gin/ginvacuum.c | 2 +- src/backend/access/gin/ginxlog.c | 33 --------------------------- src/backend/access/rmgrdesc/gindesc.c | 6 ----- src/include/access/gin.h | 3 ++- src/include/access/ginxlog.h | 2 -- 10 files changed, 27 insertions(+), 71 deletions(-) diff --git a/src/backend/access/gin/ginbtree.c b/src/backend/access/gin/ginbtree.c index 533949e46a..9f82eef8c3 100644 --- a/src/backend/access/gin/ginbtree.c +++ b/src/backend/access/gin/ginbtree.c @@ -396,7 +396,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack, /* It will fit, perform the insertion */ START_CRIT_SECTION(); - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { XLogBeginInsert(); XLogRegisterBuffer(0, stack->buffer, REGBUF_STANDARD); @@ -417,7 +417,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack, MarkBufferDirty(childbuf); } - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { XLogRecPtr recptr; ginxlogInsert xlrec; @@ -595,7 +595,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack, } /* write WAL record */ - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { XLogRecPtr recptr; diff --git a/src/backend/access/gin/gindatapage.c b/src/backend/access/gin/gindatapage.c index 3ad8b76710..a73cf944b3 100644 --- a/src/backend/access/gin/gindatapage.c +++ b/src/backend/access/gin/gindatapage.c @@ -593,7 +593,7 @@ dataBeginPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack, * Great, all the items fit on a single page. If needed, prepare data * for a WAL record describing the changes we'll make. */ - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) computeLeafRecompressWALData(leaf); /* @@ -630,6 +630,7 @@ dataBeginPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack, * subsequent insertions will probably also go to the end. This packs * the index somewhat tighter when appending to a table, which is very * common. + * */ if (!btree->isBuild) { @@ -719,7 +720,7 @@ dataExecPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack, dataPlaceToPageLeafRecompress(buf, leaf); /* If needed, register WAL data built by computeLeafRecompressWALData */ - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { XLogRegisterBufData(0, leaf->walinfo, leaf->walinfolen); } @@ -1152,7 +1153,7 @@ dataExecPlaceToPageInternal(GinBtree btree, Buffer buf, GinBtreeStack *stack, pitem = (PostingItem *) insertdata; GinDataPageAddPostingItem(page, pitem, off); - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { /* * This must be static, because it has to survive until XLogInsert, @@ -1773,6 +1774,7 @@ createPostingTree(Relation index, ItemPointerData *items, uint32 nitems, Pointer ptr; int nrootitems; int rootsize; + bool is_build = (buildStats != NULL); /* Construct the new root page in memory first. */ tmppage = (Page) palloc(BLCKSZ); @@ -1826,7 +1828,7 @@ createPostingTree(Relation index, ItemPointerData *items, uint32 nitems, PageRestoreTempPage(tmppage, page); MarkBufferDirty(buffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !is_build) { XLogRecPtr recptr; ginxlogCreatePostingTree data; diff --git a/src/backend/access/gin/ginentrypage.c b/src/backend/access/gin/ginentrypage.c index 4889de2a4f..1f5ba33d51 100644 --- a/src/backend/access/gin/ginentrypage.c +++ b/src/backend/access/gin/ginentrypage.c @@ -571,7 +571,7 @@ entryExecPlaceToPage(GinBtree btree, Buffer buf, GinBtreeStack *stack, elog(ERROR, "failed to add item to index page in \"%s\"", RelationGetRelationName(btree->index)); - if (RelationNeedsWAL(btree->index)) + if (RelationNeedsWAL(btree->index) && !btree->isBuild) { /* * This must be static, because it has to survive until XLogInsert, diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index 524ac5be8b..6307743afd 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -17,6 +17,7 @@ #include "access/gin_private.h" #include "access/ginxlog.h" #include "access/xloginsert.h" +#include "access/generic_xlog.h" #include "catalog/index.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -194,6 +195,7 @@ ginEntryInsert(GinState *ginstate, buildStats->nEntries++; ginPrepareEntryScan(&btree, attnum, key, category, ginstate); + btree.isBuild = (buildStats != NULL); stack = ginFindLeafPage(&btree, false, false, NULL); page = BufferGetPage(stack->buffer); @@ -346,23 +348,6 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo) GinInitBuffer(RootBuffer, GIN_LEAF); MarkBufferDirty(RootBuffer); - if (RelationNeedsWAL(index)) - { - XLogRecPtr recptr; - Page page; - - XLogBeginInsert(); - XLogRegisterBuffer(0, MetaBuffer, REGBUF_WILL_INIT | REGBUF_STANDARD); - XLogRegisterBuffer(1, RootBuffer, REGBUF_WILL_INIT); - - recptr = XLogInsert(RM_GIN_ID, XLOG_GIN_CREATE_INDEX); - - page = BufferGetPage(RootBuffer); - PageSetLSN(page, recptr); - - page = BufferGetPage(MetaBuffer); - PageSetLSN(page, recptr); - } UnlockReleaseBuffer(MetaBuffer); UnlockReleaseBuffer(RootBuffer); @@ -417,7 +402,16 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo) * Update metapage stats */ buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index); - ginUpdateStats(index, &buildstate.buildStats); + ginUpdateStats(index, &buildstate.buildStats, true); + + /* + * Create generic wal records for all pages of relation, if necessary. + * It seems reasonable not to generate WAL, if we recieved interrupt + * signal. + */ + CHECK_FOR_INTERRUPTS(); + if (RelationNeedsWAL(index)) + generic_log_relation(index); /* * Return statistics diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index afc20232ac..51b20bca6e 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -661,7 +661,7 @@ ginGetStats(Relation index, GinStatsData *stats) * Note: nPendingPages and ginVersion are *not* copied over */ void -ginUpdateStats(Relation index, const GinStatsData *stats) +ginUpdateStats(Relation index, const GinStatsData *stats, bool is_build) { Buffer metabuffer; Page metapage; @@ -691,7 +691,7 @@ ginUpdateStats(Relation index, const GinStatsData *stats) MarkBufferDirty(metabuffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !is_build) { XLogRecPtr recptr; ginxlogUpdateMeta data; diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index dfe885b101..b9a28d1863 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -759,7 +759,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) /* Update the metapage with accurate page and entry counts */ idxStat.nTotalPages = npages; - ginUpdateStats(info->index, &idxStat); + ginUpdateStats(info->index, &idxStat, false); /* Finally, vacuum the FSM */ IndexFreeSpaceMapVacuum(info->index); diff --git a/src/backend/access/gin/ginxlog.c b/src/backend/access/gin/ginxlog.c index c467ffa346..b648af1ff6 100644 --- a/src/backend/access/gin/ginxlog.c +++ b/src/backend/access/gin/ginxlog.c @@ -40,36 +40,6 @@ ginRedoClearIncompleteSplit(XLogReaderState *record, uint8 block_id) UnlockReleaseBuffer(buffer); } -static void -ginRedoCreateIndex(XLogReaderState *record) -{ - XLogRecPtr lsn = record->EndRecPtr; - Buffer RootBuffer, - MetaBuffer; - Page page; - - MetaBuffer = XLogInitBufferForRedo(record, 0); - Assert(BufferGetBlockNumber(MetaBuffer) == GIN_METAPAGE_BLKNO); - page = (Page) BufferGetPage(MetaBuffer); - - GinInitMetabuffer(MetaBuffer); - - PageSetLSN(page, lsn); - MarkBufferDirty(MetaBuffer); - - RootBuffer = XLogInitBufferForRedo(record, 1); - Assert(BufferGetBlockNumber(RootBuffer) == GIN_ROOT_BLKNO); - page = (Page) BufferGetPage(RootBuffer); - - GinInitBuffer(RootBuffer, GIN_LEAF); - - PageSetLSN(page, lsn); - MarkBufferDirty(RootBuffer); - - UnlockReleaseBuffer(RootBuffer); - UnlockReleaseBuffer(MetaBuffer); -} - static void ginRedoCreatePTree(XLogReaderState *record) { @@ -767,9 +737,6 @@ gin_redo(XLogReaderState *record) oldCtx = MemoryContextSwitchTo(opCtx); switch (info) { - case XLOG_GIN_CREATE_INDEX: - ginRedoCreateIndex(record); - break; case XLOG_GIN_CREATE_PTREE: ginRedoCreatePTree(record); break; diff --git a/src/backend/access/rmgrdesc/gindesc.c b/src/backend/access/rmgrdesc/gindesc.c index ef30ce16b0..f3f4e1b214 100644 --- a/src/backend/access/rmgrdesc/gindesc.c +++ b/src/backend/access/rmgrdesc/gindesc.c @@ -78,9 +78,6 @@ gin_desc(StringInfo buf, XLogReaderState *record) switch (info) { - case XLOG_GIN_CREATE_INDEX: - /* no further information */ - break; case XLOG_GIN_CREATE_PTREE: /* no further information */ break; @@ -188,9 +185,6 @@ gin_identify(uint8 info) switch (info & ~XLR_INFO_MASK) { - case XLOG_GIN_CREATE_INDEX: - id = "CREATE_INDEX"; - break; case XLOG_GIN_CREATE_PTREE: id = "CREATE_PTREE"; break; diff --git a/src/include/access/gin.h b/src/include/access/gin.h index 61fa697039..d559ffc703 100644 --- a/src/include/access/gin.h +++ b/src/include/access/gin.h @@ -71,6 +71,7 @@ extern int gin_pending_list_limit; /* ginutil.c */ extern void ginGetStats(Relation index, GinStatsData *stats); -extern void ginUpdateStats(Relation index, const GinStatsData *stats); +extern void ginUpdateStats(Relation index, + const GinStatsData *stats, bool is_build); #endif /* GIN_H */ diff --git a/src/include/access/ginxlog.h b/src/include/access/ginxlog.h index 9bd4e0b9ba..2c5d743cac 100644 --- a/src/include/access/ginxlog.h +++ b/src/include/access/ginxlog.h @@ -16,8 +16,6 @@ #include "lib/stringinfo.h" #include "storage/off.h" -#define XLOG_GIN_CREATE_INDEX 0x00 - #define XLOG_GIN_CREATE_PTREE 0x10 typedef struct ginxlogCreatePostingTree -- 2.17.1
From f032a0645d86c57e3b89a294dfb46cfad1119825 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" <a.lepik...@postgrespro.ru> Date: Tue, 5 Feb 2019 08:14:41 +0500 Subject: [PATCH 3/4] GIST-Optimal-WAL-Usage --- src/backend/access/gist/gist.c | 47 ++++++++++++++++++-------- src/backend/access/gist/gistbuild.c | 32 ++++++++++-------- src/backend/access/gist/gistutil.c | 2 +- src/backend/access/gist/gistxlog.c | 22 ------------ src/backend/access/rmgrdesc/gistdesc.c | 5 --- src/include/access/gist_private.h | 7 ++-- src/include/access/gistxlog.h | 1 - 7 files changed, 56 insertions(+), 60 deletions(-) diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index b75b3a8dac..710c1bfd3a 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -173,7 +173,7 @@ gistinsert(Relation r, Datum *values, bool *isnull, values, isnull, true /* size is currently bogus */ ); itup->t_tid = *ht_ctid; - gistdoinsert(r, itup, 0, giststate, heapRel); + gistdoinsert(r, itup, 0, giststate, heapRel, false); /* cleanup */ MemoryContextSwitchTo(oldCxt); @@ -220,7 +220,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, Buffer leftchildbuf, List **splitinfo, bool markfollowright, - Relation heapRel) + Relation heapRel, + bool is_build) { BlockNumber blkno = BufferGetBlockNumber(buffer); Page page = BufferGetPage(buffer); @@ -459,7 +460,7 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, * insertion for that. NB: The number of pages and data segments * specified here must match the calculations in gistXLogSplit()! */ - if (RelationNeedsWAL(rel)) + if (RelationNeedsWAL(rel) && !is_build) XLogEnsureRecordSpace(npage, 1 + npage * 2); START_CRIT_SECTION(); @@ -480,18 +481,20 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, PageRestoreTempPage(dist->page, BufferGetPage(dist->buffer)); dist->page = BufferGetPage(dist->buffer); - /* Write the WAL record */ - if (RelationNeedsWAL(rel)) + /* + * Write the WAL record. + * Do not write XLog entry if the insertion is caused by + * index build process. + */ + if (RelationNeedsWAL(rel) && !is_build) recptr = gistXLogSplit(is_leaf, - dist, oldrlink, oldnsn, leftchildbuf, - markfollowright); + dist, oldrlink, oldnsn, leftchildbuf, + markfollowright); else recptr = gistGetFakeLSN(rel); for (ptr = dist; ptr; ptr = ptr->next) - { PageSetLSN(ptr->page, recptr); - } /* * Return the new child buffers to the caller. @@ -545,7 +548,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, if (BufferIsValid(leftchildbuf)) MarkBufferDirty(leftchildbuf); - if (RelationNeedsWAL(rel)) + + if (RelationNeedsWAL(rel) && !is_build) { OffsetNumber ndeloffs = 0, deloffs[1]; @@ -568,6 +572,7 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, PageSetLSN(page, recptr); } + if (newblkno) *newblkno = blkno; } @@ -584,17 +589,28 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, * the full page image. There's a chicken-and-egg problem: if we updated * the child pages first, we wouldn't know the recptr of the WAL record * we're about to write. + * + * We use fakeLSNs for inserions caused by index build. And when it is + * finished, we write generic_xlog entry for each index page and update + * all LSNs. In order to keep NSNs less then LSNs after this update, we + * set NSN to InvalidXLogRecPtr, which is the smallest possible NSN. */ + if (BufferIsValid(leftchildbuf)) { Page leftpg = BufferGetPage(leftchildbuf); + XLogRecPtr fakerecptr = InvalidXLogRecPtr; - GistPageSetNSN(leftpg, recptr); - GistClearFollowRight(leftpg); + if (!is_build) + GistPageSetNSN(leftpg, recptr); + else + GistPageSetNSN(leftpg, fakerecptr); + GistClearFollowRight(leftpg); PageSetLSN(leftpg, recptr); } + END_CRIT_SECTION(); return is_split; @@ -605,9 +621,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, * this routine assumes it is invoked in a short-lived memory context, * so it does not bother releasing palloc'd allocations. */ -void gistdoinsert(Relation r, IndexTuple itup, Size freespace, - GISTSTATE *giststate, Relation heapRel) + GISTSTATE *giststate, Relation heapRel, bool is_build) { ItemId iid; IndexTuple idxtuple; @@ -620,6 +635,7 @@ gistdoinsert(Relation r, IndexTuple itup, Size freespace, state.freespace = freespace; state.r = r; state.heapRel = heapRel; + state.is_build = is_build; /* Start from the root */ firststack.blkno = GIST_ROOT_BLKNO; @@ -1237,7 +1253,8 @@ gistinserttuples(GISTInsertState *state, GISTInsertStack *stack, leftchild, &splitinfo, true, - state->heapRel); + state->heapRel, + state->is_build); /* * Before recursing up in case the page was split, release locks on the diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index bd142a3560..2e98e55187 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -17,6 +17,7 @@ #include <math.h> #include "access/genam.h" +#include "access/generic_xlog.h" #include "access/gist_private.h" #include "access/gistxlog.h" #include "access/xloginsert.h" @@ -180,18 +181,12 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) MarkBufferDirty(buffer); - if (RelationNeedsWAL(index)) - { - XLogRecPtr recptr; - - XLogBeginInsert(); - XLogRegisterBuffer(0, buffer, REGBUF_WILL_INIT); - - recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_CREATE_INDEX); - PageSetLSN(page, recptr); - } - else - PageSetLSN(page, gistGetFakeLSN(heap)); + /* + * Do not write index pages to WAL unitl index build is finished. + * But we still need increasing LSNs on each page, so use FakeLSN, + * even for relations which eventually need WAL. + */ + PageSetLSN(page, gistGetFakeLSN(heap)); UnlockReleaseBuffer(buffer); @@ -224,6 +219,15 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) freeGISTstate(buildstate.giststate); + /* + * Create generic wal records for all pages of relation, if necessary. + * It seems reasonable not to generate WAL, if we recieved interrupt + * signal. + */ + CHECK_FOR_INTERRUPTS(); + if (RelationNeedsWAL(index)) + generic_log_relation(index); + /* * Return statistics */ @@ -486,7 +490,7 @@ gistBuildCallback(Relation index, * locked, we call gistdoinsert directly. */ gistdoinsert(index, itup, buildstate->freespace, - buildstate->giststate, buildstate->heaprel); + buildstate->giststate, buildstate->heaprel, true); } /* Update tuple count and total size. */ @@ -693,7 +697,7 @@ gistbufferinginserttuples(GISTBuildState *buildstate, Buffer buffer, int level, InvalidBuffer, &splitinfo, false, - buildstate->heaprel); + buildstate->heaprel, true); /* * If this is a root split, update the root path item kept in memory. This diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index 8d3dfad27b..f7088e2d94 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -949,6 +949,7 @@ gistproperty(Oid index_oid, int attno, * Temporary and unlogged GiST indexes are not WAL-logged, but we need LSNs * to detect concurrent page splits anyway. This function provides a fake * sequence of LSNs for that purpose. + * Persistent relations are also not WAL-logged while we build index. */ XLogRecPtr gistGetFakeLSN(Relation rel) @@ -969,7 +970,6 @@ gistGetFakeLSN(Relation rel) * Unlogged relations are accessible from other backends, and survive * (clean) restarts. GetFakeLSNForUnloggedRel() handles that for us. */ - Assert(rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED); return GetFakeLSNForUnloggedRel(); } } diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index 408bd5390a..66d5bb831f 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -489,25 +489,6 @@ gistRedoPageSplitRecord(XLogReaderState *record) UnlockReleaseBuffer(firstbuffer); } -static void -gistRedoCreateIndex(XLogReaderState *record) -{ - XLogRecPtr lsn = record->EndRecPtr; - Buffer buffer; - Page page; - - buffer = XLogInitBufferForRedo(record, 0); - Assert(BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO); - page = (Page) BufferGetPage(buffer); - - GISTInitBuffer(buffer, F_LEAF); - - PageSetLSN(page, lsn); - - MarkBufferDirty(buffer); - UnlockReleaseBuffer(buffer); -} - void gist_redo(XLogReaderState *record) { @@ -532,9 +513,6 @@ gist_redo(XLogReaderState *record) case XLOG_GIST_PAGE_SPLIT: gistRedoPageSplitRecord(record); break; - case XLOG_GIST_CREATE_INDEX: - gistRedoCreateIndex(record); - break; default: elog(PANIC, "gist_redo: unknown op code %u", info); } diff --git a/src/backend/access/rmgrdesc/gistdesc.c b/src/backend/access/rmgrdesc/gistdesc.c index e468c9e15a..75f1ed747e 100644 --- a/src/backend/access/rmgrdesc/gistdesc.c +++ b/src/backend/access/rmgrdesc/gistdesc.c @@ -52,8 +52,6 @@ gist_desc(StringInfo buf, XLogReaderState *record) case XLOG_GIST_PAGE_SPLIT: out_gistxlogPageSplit(buf, (gistxlogPageSplit *) rec); break; - case XLOG_GIST_CREATE_INDEX: - break; } } @@ -73,9 +71,6 @@ gist_identify(uint8 info) case XLOG_GIST_PAGE_SPLIT: id = "PAGE_SPLIT"; break; - case XLOG_GIST_CREATE_INDEX: - id = "CREATE_INDEX"; - break; } return id; diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index 3698942f9d..b1680bfd6d 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -242,6 +242,7 @@ typedef struct Relation r; Relation heapRel; Size freespace; /* free space to be left */ + bool is_build; GISTInsertStack *stack; } GISTInsertState; @@ -391,7 +392,8 @@ extern void gistdoinsert(Relation r, IndexTuple itup, Size freespace, GISTSTATE *GISTstate, - Relation heapRel); + Relation heapRel, + bool is_build); /* A List of these is returned from gistplacetopage() in *splitinfo */ typedef struct @@ -407,7 +409,8 @@ extern bool gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, Buffer leftchildbuf, List **splitinfo, bool markleftchild, - Relation heapRel); + Relation heapRel, + bool is_build); extern SplitedPageLayout *gistSplit(Relation r, Page page, IndexTuple *itup, int len, GISTSTATE *giststate); diff --git a/src/include/access/gistxlog.h b/src/include/access/gistxlog.h index 5117aabf1a..2ee80e244e 100644 --- a/src/include/access/gistxlog.h +++ b/src/include/access/gistxlog.h @@ -22,7 +22,6 @@ /* #define XLOG_GIST_NEW_ROOT 0x20 */ /* not used anymore */ #define XLOG_GIST_PAGE_SPLIT 0x30 /* #define XLOG_GIST_INSERT_COMPLETE 0x40 */ /* not used anymore */ -#define XLOG_GIST_CREATE_INDEX 0x50 /* #define XLOG_GIST_PAGE_DELETE 0x60 */ /* not used anymore */ /* -- 2.17.1
From 3dbe3d827173244e9bda928147f35e0929266215 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" <a.lepik...@postgrespro.ru> Date: Tue, 5 Feb 2019 08:15:06 +0500 Subject: [PATCH 4/4] SPGIST-Optimal-WAL-Usage --- src/backend/access/rmgrdesc/spgdesc.c | 5 ---- src/backend/access/spgist/spgdoinsert.c | 12 ++++----- src/backend/access/spgist/spginsert.c | 24 +++-------------- src/backend/access/spgist/spgxlog.c | 35 ------------------------- src/include/access/spgxlog.h | 1 - 5 files changed, 10 insertions(+), 67 deletions(-) diff --git a/src/backend/access/rmgrdesc/spgdesc.c b/src/backend/access/rmgrdesc/spgdesc.c index 37af31a764..40c1c8b3f9 100644 --- a/src/backend/access/rmgrdesc/spgdesc.c +++ b/src/backend/access/rmgrdesc/spgdesc.c @@ -24,8 +24,6 @@ spg_desc(StringInfo buf, XLogReaderState *record) switch (info) { - case XLOG_SPGIST_CREATE_INDEX: - break; case XLOG_SPGIST_ADD_LEAF: { spgxlogAddLeaf *xlrec = (spgxlogAddLeaf *) rec; @@ -88,9 +86,6 @@ spg_identify(uint8 info) switch (info & ~XLR_INFO_MASK) { - case XLOG_SPGIST_CREATE_INDEX: - id = "CREATE_INDEX"; - break; case XLOG_SPGIST_ADD_LEAF: id = "ADD_LEAF"; break; diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c index 0d07b8b291..c34c44cd8b 100644 --- a/src/backend/access/spgist/spgdoinsert.c +++ b/src/backend/access/spgist/spgdoinsert.c @@ -289,7 +289,7 @@ addLeafTuple(Relation index, SpGistState *state, SpGistLeafTuple leafTuple, MarkBufferDirty(current->buffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; int flags; @@ -516,7 +516,7 @@ moveLeafs(Relation index, SpGistState *state, MarkBufferDirty(current->buffer); MarkBufferDirty(nbuf); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; @@ -1334,7 +1334,7 @@ doPickSplit(Relation index, SpGistState *state, saveCurrent.buffer = InvalidBuffer; } - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; int flags; @@ -1531,7 +1531,7 @@ spgAddNodeAction(Relation index, SpGistState *state, MarkBufferDirty(current->buffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; @@ -1644,7 +1644,7 @@ spgAddNodeAction(Relation index, SpGistState *state, MarkBufferDirty(saveCurrent.buffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; int flags; @@ -1840,7 +1840,7 @@ spgSplitNodeAction(Relation index, SpGistState *state, MarkBufferDirty(current->buffer); - if (RelationNeedsWAL(index)) + if (RelationNeedsWAL(index) && !state->isBuild) { XLogRecPtr recptr; diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c index f428a15138..5da0183a9b 100644 --- a/src/backend/access/spgist/spginsert.c +++ b/src/backend/access/spgist/spginsert.c @@ -21,6 +21,7 @@ #include "access/spgxlog.h" #include "access/xlog.h" #include "access/xloginsert.h" +#include "access/generic_xlog.h" #include "catalog/index.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -104,26 +105,6 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo) SpGistInitBuffer(nullbuffer, SPGIST_LEAF | SPGIST_NULLS); MarkBufferDirty(nullbuffer); - if (RelationNeedsWAL(index)) - { - XLogRecPtr recptr; - - XLogBeginInsert(); - - /* - * Replay will re-initialize the pages, so don't take full pages - * images. No other data to log. - */ - XLogRegisterBuffer(0, metabuffer, REGBUF_WILL_INIT | REGBUF_STANDARD); - XLogRegisterBuffer(1, rootbuffer, REGBUF_WILL_INIT | REGBUF_STANDARD); - XLogRegisterBuffer(2, nullbuffer, REGBUF_WILL_INIT | REGBUF_STANDARD); - - recptr = XLogInsert(RM_SPGIST_ID, XLOG_SPGIST_CREATE_INDEX); - - PageSetLSN(BufferGetPage(metabuffer), recptr); - PageSetLSN(BufferGetPage(rootbuffer), recptr); - PageSetLSN(BufferGetPage(nullbuffer), recptr); - } END_CRIT_SECTION(); @@ -150,6 +131,9 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo) SpGistUpdateMetaPage(index); + if (RelationNeedsWAL(index)) + generic_log_relation(index); + result = (IndexBuildResult *) palloc0(sizeof(IndexBuildResult)); result->heap_tuples = reltuples; result->index_tuples = buildstate.indtuples; diff --git a/src/backend/access/spgist/spgxlog.c b/src/backend/access/spgist/spgxlog.c index 71836ee8a5..ebe6ae8715 100644 --- a/src/backend/access/spgist/spgxlog.c +++ b/src/backend/access/spgist/spgxlog.c @@ -72,38 +72,6 @@ addOrReplaceTuple(Page page, Item tuple, int size, OffsetNumber offset) size); } -static void -spgRedoCreateIndex(XLogReaderState *record) -{ - XLogRecPtr lsn = record->EndRecPtr; - Buffer buffer; - Page page; - - buffer = XLogInitBufferForRedo(record, 0); - Assert(BufferGetBlockNumber(buffer) == SPGIST_METAPAGE_BLKNO); - page = (Page) BufferGetPage(buffer); - SpGistInitMetapage(page); - PageSetLSN(page, lsn); - MarkBufferDirty(buffer); - UnlockReleaseBuffer(buffer); - - buffer = XLogInitBufferForRedo(record, 1); - Assert(BufferGetBlockNumber(buffer) == SPGIST_ROOT_BLKNO); - SpGistInitBuffer(buffer, SPGIST_LEAF); - page = (Page) BufferGetPage(buffer); - PageSetLSN(page, lsn); - MarkBufferDirty(buffer); - UnlockReleaseBuffer(buffer); - - buffer = XLogInitBufferForRedo(record, 2); - Assert(BufferGetBlockNumber(buffer) == SPGIST_NULL_BLKNO); - SpGistInitBuffer(buffer, SPGIST_LEAF | SPGIST_NULLS); - page = (Page) BufferGetPage(buffer); - PageSetLSN(page, lsn); - MarkBufferDirty(buffer); - UnlockReleaseBuffer(buffer); -} - static void spgRedoAddLeaf(XLogReaderState *record) { @@ -976,9 +944,6 @@ spg_redo(XLogReaderState *record) oldCxt = MemoryContextSwitchTo(opCtx); switch (info) { - case XLOG_SPGIST_CREATE_INDEX: - spgRedoCreateIndex(record); - break; case XLOG_SPGIST_ADD_LEAF: spgRedoAddLeaf(record); break; diff --git a/src/include/access/spgxlog.h b/src/include/access/spgxlog.h index 6527fc9eb1..8199b3f250 100644 --- a/src/include/access/spgxlog.h +++ b/src/include/access/spgxlog.h @@ -18,7 +18,6 @@ #include "storage/off.h" /* XLOG record types for SPGiST */ -#define XLOG_SPGIST_CREATE_INDEX 0x00 #define XLOG_SPGIST_ADD_LEAF 0x10 #define XLOG_SPGIST_MOVE_LEAFS 0x20 #define XLOG_SPGIST_ADD_NODE 0x30 -- 2.17.1
pEpkey.asc
Description: application/pgp-keys