Hello,
I've added the updated the patch with the necessary documentation and comments.
I've referenced Robert's reply in this thread and Simon's reply in
Production block comparison facility thread to write the documentation.
This feature is used to check the consistency of WAL records, i.e,
whether the WAL records are inserted and applied correctly.
A guc parameter named wal_consistency is added to enable this feature.
When wal_consistency is enabled for a WAL record, it stores a full-page image
along with the record. When a full-page image arrives during redo, it compares
against the current page to check whether both are consistent.
The default value for this setting is none. To check all records written to the
write-ahead log, set this parameter to all. To check only some records, specify
a comma-separated list of resource managers. The resource managers which
are currently supported are xlog, heap2, heap, btree, hash, gin, gist, spgist,
sequence, brin and generic.
If any inconsistency is detected, it throws a WARNING. But, as per discussions
in the earlier threads, it can be changed to ERROR./FATAL(just a one
word change).
I've kept this as warning because of some inconsistency in BRIN VACUUM
during gmake check.
In recovery tests, I've enabled this feature in PostgresNode.pm.
Thanks to Amit, Dilip, Michael, Simon and Robert for their valuable feedbacks.
Thoughts?
--
Thanks & Regards,
Kuntal Ghosh
EnterpriseDB: http://www.enterprisedb.com
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index cd66abc..8b251b7 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -2470,6 +2470,35 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-wal-consistency" xreflabel="wal_consistency">
+ <term><varname>wal_consistency</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>wal_consistency</> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ This parameter is used to check the consistency of WAL records, i.e,
+ whether the WAL records are inserted and applied correctly. When
+ <varname>wal_consistency</varname> is enabled for a WAL record, it
+ stores a full-page image along with the record. When a full-page image
+ arrives during redo, it compares against the current page to check whether
+ both are consistent.
+ </para>
+
+ <para>
+ The default value for this setting is <literal>none</>. To check
+ all records written to the write-ahead log, set this parameter to
+ <literal>all</literal>. To check only some records, specify a
+ comma-separated list of resource managers. The resource managers
+ which are currently supported are <literal>xlog</>, <literal>heap2</>,
+ <literal>heap</>, <literal>btree</>, <literal>hash</>, <literal>gin</>,
+ <literal>gist</>, <literal>spgist</>, <literal>sequence</>, <literal>brin</>
+ and <literal>generic</>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-wal-buffers" xreflabel="wal_buffers">
<term><varname>wal_buffers</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/access/brin/brin_xlog.c b/src/backend/access/brin/brin_xlog.c
index 5a6b728..f4feb88 100644
--- a/src/backend/access/brin/brin_xlog.c
+++ b/src/backend/access/brin/brin_xlog.c
@@ -14,6 +14,7 @@
#include "access/brin_pageops.h"
#include "access/brin_xlog.h"
#include "access/xlogutils.h"
+#include "storage/bufmask.h"
/*
@@ -279,3 +280,45 @@ brin_redo(XLogReaderState *record)
elog(PANIC, "brin_redo: unknown op code %u", info);
}
}
+
+/*
+ * Mask a BRIN page
+ */
+char *
+mask_brin_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ OffsetNumber offnum,
+ maxoff;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+
+ if (BRIN_IS_REGULAR_PAGE(page_norm))
+ {
+ mask_unused_space(page_norm);
+
+ maxoff = PageGetMaxOffsetNumber(page_norm);
+ for (offnum = FirstOffsetNumber;
+ offnum <= maxoff;
+ offnum = OffsetNumberNext(offnum))
+ {
+ ItemId itemId = PageGetItemId(page_norm, offnum);
+
+ if (ItemIdIsUsed(itemId))
+ itemId->lp_flags = LP_UNUSED;
+ }
+ }
+
+ /* We need to handle brin pages of type Meta and Revmap if needed */
+
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/gin/ginxlog.c b/src/backend/access/gin/ginxlog.c
index a40f168..a247807 100644
--- a/src/backend/access/gin/ginxlog.c
+++ b/src/backend/access/gin/ginxlog.c
@@ -15,6 +15,7 @@
#include "access/gin_private.h"
#include "access/xlogutils.h"
+#include "storage/bufmask.h"
#include "utils/memutils.h"
static MemoryContext opCtx; /* working memory for operations */
@@ -758,3 +759,40 @@ gin_xlog_cleanup(void)
MemoryContextDelete(opCtx);
opCtx = NULL;
}
+
+/*
+ * Mask a Gin page
+ */
+char *
+mask_gin_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ GinPageOpaque opaque;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+ opaque = GinPageGetOpaque(page_norm);
+
+ /* GIN metapage doesn't use pd_lower/pd_upper. Other page types do. */
+ if (blkno != 0)
+ {
+ mask_page_hint_bits(page_norm);
+
+ /*
+ * For GIN_DELETED page, the page is initialized to empty.
+ * Hence mask everything.
+ */
+ if (opaque->flags & GIN_DELETED)
+ memset(page_norm, MASK_MARKER, BLCKSZ);
+ else
+ mask_unused_space(page_norm);
+ }
+
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c
index 5853d76..778b7d7 100644
--- a/src/backend/access/gist/gistxlog.c
+++ b/src/backend/access/gist/gistxlog.c
@@ -16,6 +16,7 @@
#include "access/gist_private.h"
#include "access/xloginsert.h"
#include "access/xlogutils.h"
+#include "storage/bufmask.h"
#include "utils/memutils.h"
static MemoryContext opCtx; /* working memory for operations */
@@ -443,3 +444,59 @@ gistXLogUpdate(Buffer buffer,
return recptr;
}
+
+/*
+ * Mask a GIST page
+ */
+char *
+mask_gist_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ OffsetNumber offnum,
+ maxoff;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+ mask_unused_space(page_norm);
+
+ /*Mask NSN*/
+ GistPageSetNSN(page_norm, 0xFFFFFFFFFFFFFFFF);
+
+ /*
+ * We update F_FOLLOW_RIGHT flag on the left child after writing WAL record.
+ * Hence, mask this flag.
+ */
+ GistMarkFollowRight(page_norm);
+
+ if (GistPageIsLeaf(page_norm))
+ {
+ /*
+ * For gist leaf pages,
+ * Mask some line pointer bits, particularly those marked as
+ * used on a master and unused on a standby.
+ * XXX: This could be refined.
+ */
+ maxoff = PageGetMaxOffsetNumber(page_norm);
+ for (offnum = FirstOffsetNumber;
+ offnum <= maxoff;
+ offnum = OffsetNumberNext(offnum))
+ {
+ ItemId itemId = PageGetItemId(page_norm, offnum);
+
+ if (ItemIdIsUsed(itemId))
+ itemId->lp_flags = LP_UNUSED;
+ }
+ }
+
+ /* In Gist redo, we never mark a page as garbage. Hence, Mask It.*/
+ GistClearPageHasGarbage(page_norm);
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index e3b1eef..952f7f6 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -25,6 +25,7 @@
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "optimizer/plancat.h"
+#include "storage/bufmask.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
@@ -711,3 +712,60 @@ hash_redo(XLogReaderState *record)
{
elog(PANIC, "hash_redo: unimplemented");
}
+
+/*
+ * Mask a hash page
+ */
+char *
+mask_hash_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ OffsetNumber off;
+ OffsetNumber maxoff;
+ HashPageOpaque opaque;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+ mask_unused_space(page_norm);
+
+ opaque = (HashPageOpaque) PageGetSpecialPointer(page_norm);
+ /*
+ * Mask everything on a UNUSED page.
+ */
+ if (opaque->hasho_flag & LH_UNUSED_PAGE)
+ {
+ /* Page content, between standard page header and opaque struct */
+ memset(page_norm + SizeOfPageHeaderData, MASK_MARKER,
+ BLCKSZ - MAXALIGN(sizeof(HashPageOpaqueData)) - SizeOfPageHeaderData);
+
+ /* pd_lower and upper */
+ memset(&((PageHeader) page_norm)->pd_lower, MASK_MARKER, sizeof(uint16));
+ memset(&((PageHeader) page_norm)->pd_upper, MASK_MARKER, sizeof(uint16));
+ }
+ else if ((opaque->hasho_flag & LH_META_PAGE)== 0)
+ {
+ /*
+ * For pages other than metapage,
+ * Mask some line pointer bits, particularly those marked as
+ * used on a master and unused on a standby.
+ * XXX: This could be refined.
+ */
+ maxoff = PageGetMaxOffsetNumber(page_norm);
+ for (off = 1; off <= maxoff; off++)
+ {
+ ItemId iid = PageGetItemId(page_norm, off);
+
+ if (ItemIdIsUsed(iid))
+ iid->lp_flags = LP_UNUSED;
+ }
+ }
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index b019bc1..1e80ddc 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -57,6 +57,7 @@
#include "catalog/namespace.h"
#include "miscadmin.h"
#include "pgstat.h"
+#include "storage/bufmask.h"
#include "storage/bufmgr.h"
#include "storage/freespace.h"
#include "storage/lmgr.h"
@@ -9131,3 +9132,71 @@ heap_sync(Relation rel)
heap_close(toastrel, AccessShareLock);
}
}
+
+/*
+ * Mask a heap page
+ */
+char *
+mask_heap_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ OffsetNumber off;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+ mask_unused_space(page_norm);
+
+ for (off = 1; off <= PageGetMaxOffsetNumber(page_norm); off++)
+ {
+ ItemId iid = PageGetItemId(page, off);
+ char *page_item;
+
+ page_item = (char *) (page_norm + ItemIdGetOffset(iid));
+
+ /*
+ * Ignore hint bits and command ID.
+ */
+ if (ItemIdIsNormal(iid))
+ {
+ HeapTupleHeader page_htup = (HeapTupleHeader) page_item;
+
+ page_htup->t_infomask =
+ HEAP_XMIN_COMMITTED | HEAP_XMIN_INVALID |
+ HEAP_XMAX_COMMITTED | HEAP_XMAX_INVALID;
+ page_htup->t_infomask |= HEAP_XACT_MASK;
+ page_htup->t_choice.t_heap.t_field3.t_cid = 0xFFFFFFFF;
+
+ /*
+ * For a speculative tuple, the content of t_ctid is conflicting
+ * between the backup page and current page. Hence, we set it
+ * to current block number and offset.
+ */
+ if (HeapTupleHeaderIsSpeculative(page_htup))
+ {
+ ItemPointerSet(&page_htup->t_ctid, blkno, off);
+ }
+ }
+
+ /*
+ * Ignore any padding bytes after the tuple, when the length of
+ * the item is not MAXALIGNed.
+ */
+ if (ItemIdHasStorage(iid))
+ {
+ int len = ItemIdGetLength(iid);
+ int padlen = MAXALIGN(len) - len;
+
+ if (padlen > 0)
+ memset(page_item + len, MASK_MARKER, padlen);
+ }
+ }
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c
index c536e22..cb6c96d 100644
--- a/src/backend/access/nbtree/nbtxlog.c
+++ b/src/backend/access/nbtree/nbtxlog.c
@@ -19,6 +19,7 @@
#include "access/transam.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "storage/bufmask.h"
#include "storage/procarray.h"
#include "miscadmin.h"
@@ -1028,3 +1029,64 @@ btree_redo(XLogReaderState *record)
elog(PANIC, "btree_redo: unknown op code %u", info);
}
}
+
+/*
+ * Mask a btree page
+ */
+char *
+mask_btree_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+ OffsetNumber off;
+ OffsetNumber maxoff;
+ BTPageOpaque maskopaq;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+ mask_unused_space(page_norm);
+
+ maskopaq = (BTPageOpaque)
+ (((char *) page_norm) + ((PageHeader) page_norm)->pd_special);
+ /*
+ * Mask everything on a DELETED page.
+ */
+ if (((BTPageOpaque) PageGetSpecialPointer(page_norm))->btpo_flags & BTP_DELETED)
+ {
+ /* Page content, between standard page header and opaque struct */
+ memset(page_norm + SizeOfPageHeaderData, MASK_MARKER,
+ BLCKSZ - SizeOfPageHeaderData);
+
+ /* pd_lower and upper */
+ memset(&((PageHeader) page_norm)->pd_lower, MASK_MARKER, sizeof(uint16));
+ memset(&((PageHeader) page_norm)->pd_upper, MASK_MARKER, sizeof(uint16));
+ }
+ else
+ {
+ /*
+ * Mask some line pointer bits, particularly those marked as
+ * used on a master and unused on a standby.
+ * XXX: This could be refined.
+ */
+ maxoff = PageGetMaxOffsetNumber(page_norm);
+ for (off = 1; off <= maxoff; off++)
+ {
+ ItemId iid = PageGetItemId(page_norm, off);
+
+ if (ItemIdIsUsed(iid))
+ iid->lp_flags = LP_UNUSED;
+ }
+ }
+
+ maskopaq->btpo_flags |= BTP_SPLIT_END | BTP_HAS_GARBAGE;
+ maskopaq->btpo_cycleid = 0;
+
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/spgist/spgxlog.c b/src/backend/access/spgist/spgxlog.c
index e016cdb..aa4857b 100644
--- a/src/backend/access/spgist/spgxlog.c
+++ b/src/backend/access/spgist/spgxlog.c
@@ -18,6 +18,7 @@
#include "access/transam.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "storage/bufmask.h"
#include "storage/standby.h"
#include "utils/memutils.h"
@@ -1023,3 +1024,28 @@ spg_xlog_cleanup(void)
MemoryContextDelete(opCtx);
opCtx = NULL;
}
+
+/*
+ * Mask a SpGist page
+ */
+char *
+mask_spg_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ Page page_norm;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ mask_page_hint_bits(page_norm);
+
+ if (!SpGistPageIsMeta(page_norm))
+ mask_unused_space(page_norm);
+
+ return (char *)page_norm;
+}
diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c
index 1926d98..d6b543e 100644
--- a/src/backend/access/transam/generic_xlog.c
+++ b/src/backend/access/transam/generic_xlog.c
@@ -16,6 +16,7 @@
#include "access/generic_xlog.h"
#include "access/xlogutils.h"
#include "miscadmin.h"
+#include "storage/bufmask.h"
#include "utils/memutils.h"
/*-------------------------------------------------------------------------
@@ -533,3 +534,12 @@ generic_redo(XLogReaderState *record)
UnlockReleaseBuffer(buffers[block_id]);
}
}
+
+/*
+ * Mask a generic page
+ */
+char *
+mask_generic_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ return mask_common_page(info, blkno, page, true, true);
+}
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9bb1362..3ca64d1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -26,12 +26,13 @@
#include "commands/tablespace.h"
#include "replication/message.h"
#include "replication/origin.h"
+#include "storage/bufmask.h"
#include "storage/standby.h"
#include "utils/relmapper.h"
/* must be kept in sync with RmgrData definition in xlog_internal.h */
-#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \
- { name, redo, desc, identify, startup, cleanup },
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,maskPage) \
+ { name, redo, desc, identify, startup, cleanup, maskPage },
const RmgrData RmgrTable[RM_MAX_ID + 1] = {
#include "access/rmgrlist.h"
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2189c22..f92d0ea 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -25,6 +25,7 @@
#include "access/commit_ts.h"
#include "access/multixact.h"
#include "access/rewriteheap.h"
+#include "access/rmgr.h"
#include "access/subtrans.h"
#include "access/timeline.h"
#include "access/transam.h"
@@ -52,6 +53,7 @@
#include "replication/walreceiver.h"
#include "replication/walsender.h"
#include "storage/barrier.h"
+#include "storage/bufmask.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -95,6 +97,8 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
+char *wal_consistency_string = NULL;
+bool *wal_consistency = NULL;
bool log_checkpoints = false;
int sync_method = DEFAULT_SYNC_METHOD;
int wal_level = WAL_LEVEL_MINIMAL;
@@ -870,6 +874,7 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkConsistency(XLogReaderState *record);
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6944,6 +6949,14 @@ StartupXLOG(void)
/* Now apply the WAL record itself */
RmgrTable[record->xl_rmid].rm_redo(xlogreader);
+ /*
+ * After redo, check whether the backup pages associated with the WAL record
+ * are consistent with the existing pages. This check is done only
+ * if consistency check is enabled for the corresponding rmid.
+ */
+ if (wal_consistency[record->xl_rmid])
+ checkConsistency(xlogreader);
+
/* Pop the error context stack */
error_context_stack = errcallback.previous;
@@ -11708,3 +11721,109 @@ XLogRequestWalReceiverReply(void)
{
doRequestWalReceiverReply = true;
}
+
+/*
+ * Mask a xlog page
+ */
+char *
+mask_xlog_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ /*
+ * In xlog redo, we just restore the page from backup image.
+ * Hence, we can mask it by using the common function.
+ */
+ return mask_common_page(info, blkno, page, false, false);
+}
+
+/*
+ * It checks whether the current buffer page and backup page stored in the
+ * WAL record are consistent or not. Before comparing the two pages, it applies
+ * appropiate masking to the pages to ignore certain areas like hint bits,
+ * unused space between pd_lower and pd_upper etc. For more information about
+ * masking, see the masking function.
+ * This function should be called once WAL replay has been completed.
+ */
+static void
+checkConsistency(XLogReaderState *record)
+{
+ RmgrId rmid = XLogRecGetRmid(record);
+ uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+ RelFileNode rnode;
+ ForkNumber forknum;
+ BlockNumber blkno;
+ Page new_page, old_page;
+ int block_id;
+ int inconsistent_loc;
+
+ /* Consistency is checked only for records with backup blocks*/
+ if (XLogRecHasAnyBlockRefs(record))
+ {
+ old_page = (Page) palloc(BLCKSZ);
+
+ for (block_id = 0; block_id <= record->max_block_id; block_id++)
+ {
+ Buffer buf;
+ char *norm_new_page, *norm_old_page;
+
+ if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
+ {
+ /* Caller specified a bogus block_id. Don't do anything. */
+ continue;
+ }
+ /*
+ * Read the contents from the current buffer
+ * and store it in a temporary page.
+ */
+ buf = XLogReadBufferExtended(rnode, forknum, blkno,
+ RBM_NORMAL);
+ if (!BufferIsValid(buf))
+ continue;
+ new_page = BufferGetPage(buf);
+
+ /*
+ * cross check that has_image is set if wal consistency check
+ * is enabled for current rmid.
+ */
+ if (wal_consistency[rmid] && !XLogRecHasBlockImage(record, block_id))
+ {
+ elog(ERROR,
+ "WAL consistency check is enabled, but BKPBLOCK_HAS_IMAGE not set at rel %u/%u/%u, "
+ "forknum %u, blkno %u", rnode.spcNode, rnode.dbNode, rnode.relNode,
+ forknum, blkno);
+ return;
+ }
+
+ /*
+ * Read the contents from the backup copy, stored in WAL record
+ * and store it in a temporary page.
+ */
+ if (!RestoreBlockImage(record, block_id, old_page))
+ elog(ERROR, "failed to restore block image");
+
+ /* Mask pages */
+ norm_new_page = RmgrTable[rmid].rm_maskPage(info, blkno, new_page);
+ norm_old_page = RmgrTable[rmid].rm_maskPage(info, blkno, old_page);
+
+ /* Time to compare the old and new contents */
+ inconsistent_loc = comparePages(norm_new_page, norm_old_page);
+
+ if (inconsistent_loc < BLCKSZ)
+ elog(WARNING,
+ "Inconsistent page (at byte %u) found, rel %u/%u/%u, "
+ "forknum %u, blkno %u", inconsistent_loc,
+ rnode.spcNode, rnode.dbNode, rnode.relNode,
+ forknum, blkno);
+ else
+ elog(DEBUG3,
+ "Consistent page found, rel %u/%u/%u, "
+ "forknum %u, blkno %u",
+ rnode.spcNode, rnode.dbNode, rnode.relNode,
+ forknum, blkno);
+
+ pfree(norm_new_page);
+ pfree(norm_old_page);
+ ReleaseBuffer(buf);
+ }
+ pfree(old_page);
+ }
+}
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 3cd273b..8f254b3 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -513,6 +513,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
XLogRecordBlockCompressHeader cbimg = {0};
bool samerel;
bool is_compressed = false;
+ bool include_image; /* Whether backup image should be included in WAL record */
if (!regbuf->in_use)
continue;
@@ -556,7 +557,13 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
if ((regbuf->flags & REGBUF_WILL_INIT) == REGBUF_WILL_INIT)
bkpb.fork_flags |= BKPBLOCK_WILL_INIT;
- if (needs_backup)
+ /*
+ * If needs_backup is true or wal consistency check is enabled for current rmid,
+ * we do a fpw for the current block.
+ */
+ include_image = needs_backup || wal_consistency[rmid];
+
+ if (include_image)
{
Page page = regbuf->page;
uint16 compressed_len;
@@ -618,6 +625,13 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
bimg.bimg_info = (cbimg.hole_length == 0) ? 0 : BKPIMAGE_HAS_HOLE;
+ /*
+ * Remember that, if WAL consistency check is enabled for the current rmid,
+ * we always include backup image with the WAL record. But, during redo we
+ * restore the backup block only if needs_backup is set.
+ */
+ if (needs_backup)
+ bimg.bimg_info |= BKPIMAGE_IS_REQUIRED_FOR_REDO;
if (is_compressed)
{
bimg.length = compressed_len;
@@ -680,7 +694,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
/* Ok, copy the header to the scratch buffer */
memcpy(scratch, &bkpb, SizeOfXLogRecordBlockHeader);
scratch += SizeOfXLogRecordBlockHeader;
- if (needs_backup)
+ if (include_image)
{
memcpy(scratch, &bimg, SizeOfXLogRecordBlockImageHeader);
scratch += SizeOfXLogRecordBlockImageHeader;
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 51a8e8d..e0f62ea 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -352,8 +352,8 @@ XLogReadBufferForRedoExtended(XLogReaderState *record,
if (!willinit && zeromode)
elog(PANIC, "block to be initialized in redo routine must be marked with WILL_INIT flag in the WAL record");
- /* If it's a full-page image, restore it. */
- if (XLogRecHasBlockImage(record, block_id))
+ /* If it has a full-page image for redo purpose, restore it. */
+ if (XLogRecHasBlockImageForRedo(record, block_id))
{
*buf = XLogReadBufferExtended(rnode, forknum, blkno,
get_cleanup_lock ? RBM_ZERO_AND_CLEANUP_LOCK : RBM_ZERO_AND_LOCK);
diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c
index fc3a8ee..80c6fa8 100644
--- a/src/backend/commands/sequence.c
+++ b/src/backend/commands/sequence.c
@@ -31,6 +31,7 @@
#include "funcapi.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
+#include "storage/bufmask.h"
#include "storage/lmgr.h"
#include "storage/proc.h"
#include "storage/smgr.h"
@@ -1646,3 +1647,12 @@ ResetSequenceCaches(void)
last_used_seq = NULL;
}
+
+/*
+ * Mask a Sequence page
+ */
+char *
+mask_seq_page(uint8 info, BlockNumber blkno, const char *page)
+{
+ return mask_common_page(info, blkno, page, false, true);
+}
diff --git a/src/backend/storage/buffer/Makefile b/src/backend/storage/buffer/Makefile
index 2c10fba..8630dca 100644
--- a/src/backend/storage/buffer/Makefile
+++ b/src/backend/storage/buffer/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/storage/buffer
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
-OBJS = buf_table.o buf_init.o bufmgr.o freelist.o localbuf.o
+OBJS = buf_table.o buf_init.o bufmask.o bufmgr.o freelist.o localbuf.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/storage/buffer/bufmask.c b/src/backend/storage/buffer/bufmask.c
new file mode 100644
index 0000000..0270be6
--- /dev/null
+++ b/src/backend/storage/buffer/bufmask.c
@@ -0,0 +1,105 @@
+/*-------------------------------------------------------------------------
+ *
+ * bufmask.c
+ * Routines for buffer masking, used to ensure that buffers used for
+ * comparison across nodes are in a consistent state.
+ *
+ * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * Contains common routines required for masking a page.
+ *
+ * IDENTIFICATION
+ * src/backend/storage/buffer/bufmask.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "storage/bufmask.h"
+
+/*
+ * Mask Page LSN
+ */
+void
+mask_page_lsn(Page page)
+{
+ PageHeader phdr = (PageHeader) page;
+ PageXLogRecPtrSet(phdr->pd_lsn, 0xFFFFFFFFFFFFFFFF);
+}
+
+/*
+ * Mask hint bits in PageHeader
+ */
+void
+mask_page_hint_bits(Page page)
+{
+ PageHeader phdr = (PageHeader) page;
+
+ /* Ignore prune_xid (it's like a hint-bit) */
+ phdr->pd_prune_xid = 0xFFFFFFFF;
+
+ /* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints */
+ phdr->pd_flags |= PD_PAGE_FULL | PD_HAS_FREE_LINES;
+
+ /*
+ * Also mask the all-visible flag.
+ *
+ * XXX: It is unfortunate that we have to do this. If the flag is set
+ * incorrectly, that's serious, and we would like to catch it. If the flag
+ * is cleared incorrectly, that's serious too. But redo of HEAP_CLEAN
+ * records don't currently set the flag, even though it is set in the
+ * master, so we must silence failures that that causes.
+ */
+ phdr->pd_flags |= PD_ALL_VISIBLE;
+}
+
+/*
+ * Mask the unused space of a page between pd_lower and pd_upper.
+ */
+void
+mask_unused_space(Page page)
+{
+ int pd_lower = ((PageHeader) page)->pd_lower;
+ int pd_upper = ((PageHeader) page)->pd_upper;
+ int pd_special = ((PageHeader) page)->pd_special;
+
+ /* Sanity check */
+ if (pd_lower > pd_upper || pd_special < pd_upper ||
+ pd_lower < SizeOfPageHeaderData || pd_special > BLCKSZ)
+ {
+ elog(ERROR, "invalid page pd_lower %u pd_upper %u pd_special %u\n",
+ pd_lower, pd_upper, pd_special);
+ }
+
+ memset(page + pd_lower, MASK_MARKER, pd_upper - pd_lower);
+}
+
+/*
+ * Mask a common page, i.e., mask the lsn, hint bits, and unused space between
+ * pd_lower and pd_upper. Although, hint bits and unused space can be masked
+ * optionally.
+ */
+char *
+mask_common_page(uint8 info, BlockNumber blkno, const char *page, bool maskHints, bool maskUnusedSpace)
+{
+ Page page_norm;
+
+ page_norm = (Page) palloc(BLCKSZ);
+ memcpy(page_norm, page, BLCKSZ);
+
+ /*
+ * Mask the Page LSN. Because, we store the page before updating the LSN.
+ * Hence, LSNs of both pages will always be different.
+ */
+ mask_page_lsn(page_norm);
+
+ if (maskHints)
+ mask_page_hint_bits(page_norm);
+
+ if (maskUnusedSpace)
+ mask_unused_space(page_norm);
+
+ return (char *)page_norm;
+}
diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c
index 73aa0c0..877b58d 100644
--- a/src/backend/storage/page/bufpage.c
+++ b/src/backend/storage/page/bufpage.c
@@ -1177,3 +1177,23 @@ PageSetChecksumInplace(Page page, BlockNumber blkno)
((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno);
}
+
+/*
+ * Compare the contents of two pages.
+ * If the two pages are exactly same, it returns BLCKSZ. Otherwise,
+ * it returns the byte location where the first mismatch has occurred.
+ */
+int
+comparePages(char *page1, char *page2)
+{
+ int i;
+
+ for (i = 0; i < BLCKSZ ; i++)
+ {
+ uint8 byte1 = (uint8) page1[i];
+ uint8 byte2 = (uint8) page2[i];
+ if(byte1 != byte2)
+ break;
+ }
+ return i;
+}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index c72bd61..14128a5 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -28,9 +28,11 @@
#include "access/commit_ts.h"
#include "access/gin.h"
+#include "access/rmgr.h"
#include "access/transam.h"
#include "access/twophase.h"
#include "access/xact.h"
+#include "access/xlog_internal.h"
#include "catalog/namespace.h"
#include "commands/async.h"
#include "commands/prepare.h"
@@ -144,6 +146,9 @@ static bool call_enum_check_hook(struct config_enum * conf, int *newval,
static bool check_log_destination(char **newval, void **extra, GucSource source);
static void assign_log_destination(const char *newval, void *extra);
+static bool check_wal_consistency(char **newval, void **extra, GucSource source);
+static void assign_wal_consistency(const char *newval, void *extra);
+
#ifdef HAVE_SYSLOG
static int syslog_facility = LOG_LOCAL0;
#else
@@ -3248,6 +3253,17 @@ static struct config_string ConfigureNamesString[] =
},
{
+ {"wal_consistency", PGC_POSTMASTER, WAL_SETTINGS,
+ gettext_noop("Sets the rmgrIDs for which WAL consistency should be checked."),
+ gettext_noop("Valid values are combinations of rmgrIDs"),
+ GUC_LIST_INPUT
+ },
+ &wal_consistency_string,
+ "none",
+ check_wal_consistency, assign_wal_consistency, NULL
+ },
+
+ {
{"log_destination", PGC_SIGHUP, LOGGING_WHERE,
gettext_noop("Sets the destination for server log output."),
gettext_noop("Valid values are combinations of \"stderr\", "
@@ -9903,6 +9919,122 @@ assign_log_destination(const char *newval, void *extra)
Log_destination = *((int *) extra);
}
+static bool
+check_wal_consistency(char **newval, void **extra, GucSource source)
+{
+ char *rawstring;
+ List *elemlist;
+ ListCell *l;
+ bool *newwalconsistency;
+ bool isRmgrId = false; /* Does this guc include any individual rmid? */
+ bool isAll = false; /* Does this guc include 'all' keyword? */
+ bool isNone = false; /* Does this guc include 'none' keyword? */
+ int i;
+
+ newwalconsistency = (bool *) guc_malloc(ERROR, (RM_MAX_ID + 1) * sizeof(bool));
+
+ /* Initialize the array*/
+ MemSet(newwalconsistency, 0, (RM_MAX_ID + 1) * sizeof(bool));
+
+ /* Need a modifiable copy of string */
+ rawstring = pstrdup(*newval);
+
+ /* Parse string into list of identifiers */
+ if (!SplitIdentifierString(rawstring, ',', &elemlist))
+ {
+ /* syntax error in list */
+ GUC_check_errdetail("List syntax is invalid.");
+ pfree(rawstring);
+ list_free(elemlist);
+ return false;
+ }
+
+ foreach(l, elemlist)
+ {
+ char *tok = (char *) lfirst(l);
+ bool found = false;
+
+ /* Check if the token matches with any individual rmid */
+ for (i = 0; i <= RM_MAX_ID; i++)
+ {
+ if (pg_strcasecmp(tok, RmgrTable[i].rm_name) == 0)
+ {
+ /*
+ * Found a match. Now, check if maskPage function
+ * is defined for this rmid. We'll enable this feature
+ * only for the rmids for which a masking function is defined.
+ */
+ if (RmgrTable[i].rm_maskPage != NULL)
+ {
+ newwalconsistency[i] = true;
+ found = true;
+ isRmgrId = true;
+ break;
+ }
+ else
+ {
+ GUC_check_errdetail("Unrecognized key word: \"%s\".", tok);
+ pfree(rawstring);
+ list_free(elemlist);
+ return false;
+ }
+ }
+ }
+ /* If a valid rmid is found, check for the next one. */
+ if (found)
+ continue;
+
+ /* Definitely not an individual rmid. Check for 'none' and 'all'. */
+ if (pg_strcasecmp(tok, "none") == 0)
+ {
+ MemSet(newwalconsistency, 0, (RM_MAX_ID + 1) * sizeof(bool));
+ isNone = true;
+ }
+ else if (pg_strcasecmp(tok, "all") == 0)
+ {
+ /*
+ * We'll enable this feature only for the rmids for which
+ * a masking function is defined.
+ */
+ for (i = 0; i <= RM_MAX_ID; i++)
+ {
+ if (RmgrTable[i].rm_maskPage != NULL)
+ {
+ newwalconsistency[i] = true;
+ }
+ }
+ isAll = true;
+ }
+ else
+ {
+ GUC_check_errdetail("Unrecognized key word: \"%s\".", tok);
+ pfree(rawstring);
+ list_free(elemlist);
+ return false;
+ }
+ }
+
+ pfree(rawstring);
+ list_free(elemlist);
+
+ /* guc should contain either 'all' or 'none' or combination of rmids. */
+ if ((isAll && isNone) || (isAll && isRmgrId) || (isNone && isRmgrId))
+ {
+ GUC_check_errdetail("Invalid value combination");
+ return false;
+ }
+
+ *extra = (void *) newwalconsistency;
+
+ return true;
+}
+
+static void
+assign_wal_consistency(const char *newval, void *extra)
+{
+ wal_consistency = (bool *) extra;
+}
+
static void
assign_syslog_facility(int newval, void *extra)
{
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b1c3aea..93041a1 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -191,6 +191,11 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
+#wal_consistency = 'none' # Valid values are combinations of
+ # heap2, heap, btree, hash, gin, gist, sequence,
+ # spgist, brin, generic and xlog. It can also
+ # be set to all to enable all the values.
+ # (change requires restart)
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index b53591d..baeeecc 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -29,7 +29,7 @@
* RmgrNames is an array of resource manager names, to make error messages
* a bit nicer.
*/
-#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,maskPage) \
name,
static const char *RmgrNames[RM_MAX_ID + 1] = {
diff --git a/src/bin/pg_xlogdump/rmgrdesc.c b/src/bin/pg_xlogdump/rmgrdesc.c
index 8fe20ce..f962e79 100644
--- a/src/bin/pg_xlogdump/rmgrdesc.c
+++ b/src/bin/pg_xlogdump/rmgrdesc.c
@@ -32,7 +32,7 @@
#include "storage/standbydefs.h"
#include "utils/relmapper.h"
-#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,maskPage) \
{ name, desc, identify},
const RmgrDescData RmgrDescTable[RM_MAX_ID + 1] = {
diff --git a/src/include/access/brin_xlog.h b/src/include/access/brin_xlog.h
index f614805..6c53b3f 100644
--- a/src/include/access/brin_xlog.h
+++ b/src/include/access/brin_xlog.h
@@ -128,5 +128,6 @@ typedef struct xl_brin_revmap_extend
extern void brin_redo(XLogReaderState *record);
extern void brin_desc(StringInfo buf, XLogReaderState *record);
extern const char *brin_identify(uint8 info);
+extern char *mask_brin_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* BRIN_XLOG_H */
diff --git a/src/include/access/generic_xlog.h b/src/include/access/generic_xlog.h
index 63f2120..16135e1 100644
--- a/src/include/access/generic_xlog.h
+++ b/src/include/access/generic_xlog.h
@@ -40,5 +40,6 @@ extern void GenericXLogAbort(GenericXLogState *state);
extern void generic_redo(XLogReaderState *record);
extern const char *generic_identify(uint8 info);
extern void generic_desc(StringInfo buf, XLogReaderState *record);
+extern char *mask_generic_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* GENERIC_XLOG_H */
diff --git a/src/include/access/gin.h b/src/include/access/gin.h
index e5b2e10..a359545 100644
--- a/src/include/access/gin.h
+++ b/src/include/access/gin.h
@@ -79,5 +79,6 @@ extern void gin_desc(StringInfo buf, XLogReaderState *record);
extern const char *gin_identify(uint8 info);
extern void gin_xlog_startup(void);
extern void gin_xlog_cleanup(void);
+extern char *mask_gin_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* GIN_H */
diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h
index 1231585..787a643 100644
--- a/src/include/access/gist_private.h
+++ b/src/include/access/gist_private.h
@@ -464,6 +464,7 @@ extern void gist_desc(StringInfo buf, XLogReaderState *record);
extern const char *gist_identify(uint8 info);
extern void gist_xlog_startup(void);
extern void gist_xlog_cleanup(void);
+extern char *mask_gist_page(uint8 info, BlockNumber blkno, const char *page);
extern XLogRecPtr gistXLogUpdate(Buffer buffer,
OffsetNumber *todelete, int ntodelete,
diff --git a/src/include/access/hash_xlog.h b/src/include/access/hash_xlog.h
index 5f941a9..54780f2 100644
--- a/src/include/access/hash_xlog.h
+++ b/src/include/access/hash_xlog.h
@@ -21,5 +21,6 @@
extern void hash_redo(XLogReaderState *record);
extern void hash_desc(StringInfo buf, XLogReaderState *record);
extern const char *hash_identify(uint8 info);
+extern char *mask_hash_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* HASH_XLOG_H */
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 06a8242..afffc26 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -373,6 +373,7 @@ extern void HeapTupleHeaderAdvanceLatestRemovedXid(HeapTupleHeader tuple,
extern void heap_redo(XLogReaderState *record);
extern void heap_desc(StringInfo buf, XLogReaderState *record);
extern const char *heap_identify(uint8 info);
+extern char *mask_heap_page(uint8 info, BlockNumber blkno, const char *page);
extern void heap2_redo(XLogReaderState *record);
extern void heap2_desc(StringInfo buf, XLogReaderState *record);
extern const char *heap2_identify(uint8 info);
diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h
index c580f51..abaf275 100644
--- a/src/include/access/nbtree.h
+++ b/src/include/access/nbtree.h
@@ -775,5 +775,6 @@ extern void _bt_leafbuild(BTSpool *btspool, BTSpool *spool2);
extern void btree_redo(XLogReaderState *record);
extern void btree_desc(StringInfo buf, XLogReaderState *record);
extern const char *btree_identify(uint8 info);
+extern char *mask_btree_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* NBTREE_H */
diff --git a/src/include/access/rmgr.h b/src/include/access/rmgr.h
index ff7fe62..0d2bc1a 100644
--- a/src/include/access/rmgr.h
+++ b/src/include/access/rmgr.h
@@ -19,7 +19,7 @@ typedef uint8 RmgrId;
* Note: RM_MAX_ID must fit in RmgrId; widening that type will affect the XLOG
* file format.
*/
-#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,maskPage) \
symname,
typedef enum RmgrIds
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index a7a0ae2..9e693b4 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -25,25 +25,25 @@
*/
/* symbol name, textual name, redo, desc, identify, startup, cleanup */
-PG_RMGR(RM_XLOG_ID, "XLOG", xlog_redo, xlog_desc, xlog_identify, NULL, NULL)
-PG_RMGR(RM_XACT_ID, "Transaction", xact_redo, xact_desc, xact_identify, NULL, NULL)
-PG_RMGR(RM_SMGR_ID, "Storage", smgr_redo, smgr_desc, smgr_identify, NULL, NULL)
-PG_RMGR(RM_CLOG_ID, "CLOG", clog_redo, clog_desc, clog_identify, NULL, NULL)
-PG_RMGR(RM_DBASE_ID, "Database", dbase_redo, dbase_desc, dbase_identify, NULL, NULL)
-PG_RMGR(RM_TBLSPC_ID, "Tablespace", tblspc_redo, tblspc_desc, tblspc_identify, NULL, NULL)
-PG_RMGR(RM_MULTIXACT_ID, "MultiXact", multixact_redo, multixact_desc, multixact_identify, NULL, NULL)
-PG_RMGR(RM_RELMAP_ID, "RelMap", relmap_redo, relmap_desc, relmap_identify, NULL, NULL)
-PG_RMGR(RM_STANDBY_ID, "Standby", standby_redo, standby_desc, standby_identify, NULL, NULL)
-PG_RMGR(RM_HEAP2_ID, "Heap2", heap2_redo, heap2_desc, heap2_identify, NULL, NULL)
-PG_RMGR(RM_HEAP_ID, "Heap", heap_redo, heap_desc, heap_identify, NULL, NULL)
-PG_RMGR(RM_BTREE_ID, "Btree", btree_redo, btree_desc, btree_identify, NULL, NULL)
-PG_RMGR(RM_HASH_ID, "Hash", hash_redo, hash_desc, hash_identify, NULL, NULL)
-PG_RMGR(RM_GIN_ID, "Gin", gin_redo, gin_desc, gin_identify, gin_xlog_startup, gin_xlog_cleanup)
-PG_RMGR(RM_GIST_ID, "Gist", gist_redo, gist_desc, gist_identify, gist_xlog_startup, gist_xlog_cleanup)
-PG_RMGR(RM_SEQ_ID, "Sequence", seq_redo, seq_desc, seq_identify, NULL, NULL)
-PG_RMGR(RM_SPGIST_ID, "SPGist", spg_redo, spg_desc, spg_identify, spg_xlog_startup, spg_xlog_cleanup)
-PG_RMGR(RM_BRIN_ID, "BRIN", brin_redo, brin_desc, brin_identify, NULL, NULL)
-PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_identify, NULL, NULL)
-PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL)
-PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL)
-PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL)
+PG_RMGR(RM_XLOG_ID, "XLOG", xlog_redo, xlog_desc, xlog_identify, NULL, NULL, mask_xlog_page)
+PG_RMGR(RM_XACT_ID, "Transaction", xact_redo, xact_desc, xact_identify, NULL, NULL, NULL)
+PG_RMGR(RM_SMGR_ID, "Storage", smgr_redo, smgr_desc, smgr_identify, NULL, NULL, NULL)
+PG_RMGR(RM_CLOG_ID, "CLOG", clog_redo, clog_desc, clog_identify, NULL, NULL, NULL)
+PG_RMGR(RM_DBASE_ID, "Database", dbase_redo, dbase_desc, dbase_identify, NULL, NULL, NULL)
+PG_RMGR(RM_TBLSPC_ID, "Tablespace", tblspc_redo, tblspc_desc, tblspc_identify, NULL, NULL, NULL)
+PG_RMGR(RM_MULTIXACT_ID, "MultiXact", multixact_redo, multixact_desc, multixact_identify, NULL, NULL, NULL)
+PG_RMGR(RM_RELMAP_ID, "RelMap", relmap_redo, relmap_desc, relmap_identify, NULL, NULL, NULL)
+PG_RMGR(RM_STANDBY_ID, "Standby", standby_redo, standby_desc, standby_identify, NULL, NULL, NULL)
+PG_RMGR(RM_HEAP2_ID, "Heap2", heap2_redo, heap2_desc, heap2_identify, NULL, NULL, mask_heap_page)
+PG_RMGR(RM_HEAP_ID, "Heap", heap_redo, heap_desc, heap_identify, NULL, NULL, mask_heap_page)
+PG_RMGR(RM_BTREE_ID, "Btree", btree_redo, btree_desc, btree_identify, NULL, NULL, mask_btree_page)
+PG_RMGR(RM_HASH_ID, "Hash", hash_redo, hash_desc, hash_identify, NULL, NULL, mask_hash_page)
+PG_RMGR(RM_GIN_ID, "Gin", gin_redo, gin_desc, gin_identify, gin_xlog_startup, gin_xlog_cleanup, mask_gin_page)
+PG_RMGR(RM_GIST_ID, "Gist", gist_redo, gist_desc, gist_identify, gist_xlog_startup, gist_xlog_cleanup, mask_gist_page)
+PG_RMGR(RM_SEQ_ID, "Sequence", seq_redo, seq_desc, seq_identify, NULL, NULL, mask_seq_page)
+PG_RMGR(RM_SPGIST_ID, "SPGist", spg_redo, spg_desc, spg_identify, spg_xlog_startup, spg_xlog_cleanup, mask_spg_page)
+PG_RMGR(RM_BRIN_ID, "BRIN", brin_redo, brin_desc, brin_identify, NULL, NULL, mask_brin_page)
+PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_identify, NULL, NULL, NULL)
+PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
+PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, mask_generic_page)
+PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
diff --git a/src/include/access/spgist.h b/src/include/access/spgist.h
index a953a5a..6e52ea3 100644
--- a/src/include/access/spgist.h
+++ b/src/include/access/spgist.h
@@ -220,5 +220,6 @@ extern void spg_desc(StringInfo buf, XLogReaderState *record);
extern const char *spg_identify(uint8 info);
extern void spg_xlog_startup(void);
extern void spg_xlog_cleanup(void);
+extern char *mask_spg_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* SPGIST_H */
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index c9f332c..0238d21 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -105,6 +105,8 @@ extern bool EnableHotStandby;
extern bool fullPageWrites;
extern bool wal_log_hints;
extern bool wal_compression;
+extern bool *wal_consistency;
+extern char *wal_consistency_string;
extern bool log_checkpoints;
extern int CheckPointSegments;
@@ -226,6 +228,7 @@ extern void XLogSetReplicationSlotMinimumLSN(XLogRecPtr lsn);
extern void xlog_redo(XLogReaderState *record);
extern void xlog_desc(StringInfo buf, XLogReaderState *record);
extern const char *xlog_identify(uint8 info);
+extern char *mask_xlog_page(uint8 info, BlockNumber blkno, const char *page);
extern void issue_xlog_fsync(int fd, XLogSegNo segno);
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 0a595cc..47fb0d0 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -276,6 +276,8 @@ typedef struct RmgrData
const char *(*rm_identify) (uint8 info);
void (*rm_startup) (void);
void (*rm_cleanup) (void);
+ char *(*rm_maskPage) (uint8 info, BlockNumber blkno,
+ const char *page);
} RmgrData;
extern const RmgrData RmgrTable[];
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index deaa7f5..5112e60 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -205,6 +205,9 @@ extern bool DecodeXLogRecord(XLogReaderState *state, XLogRecord *record,
((decoder)->blocks[block_id].in_use)
#define XLogRecHasBlockImage(decoder, block_id) \
((decoder)->blocks[block_id].has_image)
+#define XLogRecHasBlockImageForRedo(decoder, block_id) \
+ (XLogRecHasBlockImage(decoder, block_id) && \
+ (((decoder)->blocks[block_id].bimg_info & BKPIMAGE_IS_REQUIRED_FOR_REDO) > 0))
extern bool RestoreBlockImage(XLogReaderState *recoder, uint8 block_id, char *dst);
extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len);
diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h
index 3dfcb49..d747ab1 100644
--- a/src/include/access/xlogrecord.h
+++ b/src/include/access/xlogrecord.h
@@ -137,6 +137,7 @@ typedef struct XLogRecordBlockImageHeader
/* Information stored in bimg_info */
#define BKPIMAGE_HAS_HOLE 0x01 /* page image has "hole" */
#define BKPIMAGE_IS_COMPRESSED 0x02 /* page image is compressed */
+#define BKPIMAGE_IS_REQUIRED_FOR_REDO 0x04 /* page is required during redo */
/*
* Extra header information used when page image has "hole" and
diff --git a/src/include/commands/sequence.h b/src/include/commands/sequence.h
index 392a626..a26102f 100644
--- a/src/include/commands/sequence.h
+++ b/src/include/commands/sequence.h
@@ -82,5 +82,6 @@ extern void ResetSequenceCaches(void);
extern void seq_redo(XLogReaderState *rptr);
extern void seq_desc(StringInfo buf, XLogReaderState *rptr);
extern const char *seq_identify(uint8 info);
+extern char *mask_seq_page(uint8 info, BlockNumber blkno, const char *page);
#endif /* SEQUENCE_H */
diff --git a/src/include/storage/bufmask.h b/src/include/storage/bufmask.h
new file mode 100644
index 0000000..0af9c35
--- /dev/null
+++ b/src/include/storage/bufmask.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * bufmask.h
+ * Buffer masking definitions.
+ *
+ * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/storage/bufmask.h
+ */
+
+#ifndef BUFMASK_H
+#define BUFMASK_H
+
+#include "postgres.h"
+#include "storage/block.h"
+#include "storage/bufmgr.h"
+
+/* Marker used to mask pages consistently */
+#define MASK_MARKER 0xFF
+
+extern void mask_page_lsn(Page page);
+extern void mask_page_hint_bits(Page page);
+extern void mask_unused_space(Page page);
+extern char *mask_common_page(uint8 info, BlockNumber blkno,
+ const char *page, bool maskHints,
+ bool maskUnusedSpace);
+#endif
diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h
index ad4ab5f..a5f34d3 100644
--- a/src/include/storage/bufpage.h
+++ b/src/include/storage/bufpage.h
@@ -435,4 +435,6 @@ extern bool PageIndexTupleOverwrite(Page page, OffsetNumber offnum,
extern char *PageSetChecksumCopy(Page page, BlockNumber blkno);
extern void PageSetChecksumInplace(Page page, BlockNumber blkno);
+extern int comparePages(Page norm_new_page, Page norm_old_page);
+
#endif /* BUFPAGE_H */
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index fede1e6..5ef703e 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -404,6 +404,7 @@ sub init
print $conf "fsync = off\n";
print $conf "log_statement = all\n";
print $conf "port = $port\n";
+ print $conf "wal_consistency = all\n";
if ($params{allows_streaming})
{
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
