Hi, hackers!
I want to propose adding hook in BufferSync.
==Why==
So that extensions could track pages changed between checkpoints. I think this can allow efficient differential backups taken right after checkpoint. And this functionality can be implemented as an extension.
==What==
I propose to add hook inside BufferSync() function it inform extensions that we are going to write pages to disk. Please see patch attached. I pass a timestamp of the checkpoint, but it would be good if we could also pass there number of checkpoint or something like this to ensure that some checkpoints were not lost (this could yield malformed backups).
==State==
This is just an idea to discuss, I could not find something like this in pgsql-hackers as for now. Neither I could find similar hooks in the code.
Is this hook sufficient to implement page tracking for differential backups? I’m not sure, but seems like it is.
==Questions==
Is this call enough to gather information about changed pages for backup purposes?
Can we call extensions reliably from checkpointer process?
Can we guaranty that extension won’t miss our call or we will defer BufferSync if extension have failed?
Can we provide more flexibility for this hook?
Any thought will be appreciated.
Best regards, Andrey Borodin, Yandex.
From 01d692e01716d6904847e4ce73faabbd7cc9fa97 Mon Sep 17 00:00:00 2001 From: Andrey Borodin <amboro...@acm.org> Date: Sat, 5 Aug 2017 12:36:32 +0500 Subject: [PATCH] Add hook to BuferSync
---
src/backend/storage/buffer/bufmgr.c | 37 +++++++++--------------------------
src/include/storage/buf_internals.h | 39 +++++++++++++++++++++++++++++++++++--
2 files changed, 46 insertions(+), 30 deletions(-)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 15795b0c5a..57bb5b89f0 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -76,34 +76,6 @@ typedef struct PrivateRefCountEntry
/* 64 bytes, about the size of a cache line on common systems */
#define REFCOUNT_ARRAY_ENTRIES 8
-/*
- * Status of buffers to checkpoint for a particular tablespace, used
- * internally in BufferSync.
- */
-typedef struct CkptTsStatus
-{
- /* oid of the tablespace */
- Oid tsId;
-
- /*
- * Checkpoint progress for this tablespace. To make progress comparable
- * between tablespaces the progress is, for each tablespace, measured as a
- * number between 0 and the total number of to-be-checkpointed pages. Each
- * page checkpointed in this tablespace increments this space's progress
- * by progress_slice.
- */
- float8 progress;
- float8 progress_slice;
-
- /* number of to-be checkpointed pages in this tablespace */
- int num_to_scan;
- /* already processed pages in this tablespace */
- int num_scanned;
-
- /* current offset in CkptBufferIds for this tablespace */
- int index;
-} CkptTsStatus;
-
/* GUC variables */
bool zero_damaged_pages = false;
int bgwriter_lru_maxpages = 100;
@@ -1764,6 +1736,10 @@ UnpinBuffer(BufferDesc *buf, bool fixOwner)
}
}
+
+/* Hook for plugins to get control in BufferSync() */
+checkpointer_buffer_sync_hook_type checkpointer_buffer_sync_hook = NULL;
+
/*
* BufferSync -- Write out all dirty buffers in the pool.
*
@@ -1926,6 +1902,11 @@ BufferSync(int flags)
Assert(num_spaces > 0);
+ if(checkpointer_buffer_sync_hook)
+ checkpointer_buffer_sync_hook(CheckpointStats.ckpt_write_t,
+ CkptBufferIds,
+ per_ts_stat);
+
/*
* Build a min-heap over the write-progress in the individual tablespaces,
* and compute how large a portion of the total progress a single
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index b768b6fc96..b9e4f19f3a 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -278,8 +278,6 @@ extern PGDLLIMPORT WritebackContext BackendWritebackContext;
/* in localbuf.c */
extern BufferDesc *LocalBufferDescriptors;
-/* in bufmgr.c */
-
/*
* Structure to sort buffers per file on checkpoints.
*
@@ -295,9 +293,46 @@ typedef struct CkptSortItem
int buf_id;
} CkptSortItem;
+/* in bufmgr.c */
extern CkptSortItem *CkptBufferIds;
/*
+ * Status of buffers to checkpoint for a particular tablespace, used
+ * internally in BufferSync.
+ */
+typedef struct CkptTsStatus
+{
+ /* oid of the tablespace */
+ Oid tsId;
+
+ /*
+ * Checkpoint progress for this tablespace. To make progress comparable
+ * between tablespaces the progress is, for each tablespace, measured as a
+ * number between 0 and the total number of to-be-checkpointed pages. Each
+ * page checkpointed in this tablespace increments this space's progress
+ * by progress_slice.
+ */
+ float8 progress;
+ float8 progress_slice;
+
+ /* number of to-be checkpointed pages in this tablespace */
+ int num_to_scan;
+ /* already processed pages in this tablespace */
+ int num_scanned;
+
+ /* current offset in CkptBufferIds for this tablespace */
+ int index;
+} CkptTsStatus;
+
+/* Hook for plugins to get control in BufferSync() */
+typedef void (*checkpointer_buffer_sync_hook_type) (TimestampTz ckpt_write_t,
+ CkptSortItem *ckptBufferIds,
+ CkptTsStatus *per_ts_stat);
+
+/* in bufmgr.c */
+extern PGDLLIMPORT checkpointer_buffer_sync_hook_type checkpointer_buffer_sync_hook;
+
+/*
* Internal buffer management routines
*/
/* bufmgr.c */
--
2.11.0 (Apple Git-81)
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
