Hi,
Some people want to disable xlog archiving only during synchronous
replication. So, first, I created and attached the self-contained patch
to control xlog archiving dynamically. Since it would still take time to
fix whole patch of synch rep, and opening it at a time would burden
reviewers, I opened this patch this time.
http://archives.postgresql.org/pgsql-hackers/2008-12/msg00718.php
This patch provides three abilities:
(1)
Turn off xlog archiving from the specified file. For example, if the
xlog file including replication starting position is specified, we can
stop archiving the xlog files which are generated during replication.
Specifically, we only skip executing archive command after turning
off, that is, archiver is still in progress and .done file is created.
This reduces the burden of resuming archiving.
(2)
Turn back on xlog archiving from the specified file.
(3)
Revert the arhive status file of the specified file from .done to .ready.
This would try to archive the corresponding xlog file again. The archive
status of only skipped xlog file can be reverted.
We resume xlog archiving by using (2) and (3), when replication
ends.
(1) and (2) are provided as PgArchControl() in pgarch.c.
(3) is provided as XLogArchiveRevert() in xlog.c.
Please feel free to comment!
Regards,
--
Fujii Masao
NIPPON TELEGRAPH AND TELEPHONE CORPORATION
NTT Open Source Software Center
? src/backend/postmaster/walreceiver.c
? src/backend/postmaster/walsender.c
? src/include/postmaster/walreceiver.h
? src/include/postmaster/walsender.h
Index: src/backend/access/transam/xlog.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/access/transam/xlog.c,v
retrieving revision 1.324
diff -c -r1.324 xlog.c
*** src/backend/access/transam/xlog.c 17 Dec 2008 01:39:03 -0000 1.324
--- src/backend/access/transam/xlog.c 24 Dec 2008 05:24:50 -0000
***************
*** 38,43 ****
--- 38,44 ----
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/bgwriter.h"
+ #include "postmaster/pgarch.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
***************
*** 1130,1135 ****
--- 1131,1205 ----
}
/*
+ * XLogArchiveRevert
+ *
+ * Revert a particular status file from .done to .ready. This
+ * would try to archive the corresponding xlog file again.
+ *
+ * We can revert only the status file of xlog which we skipped
+ * archiving before. And, if there is no particular status file,
+ * we do nothing.
+ *
+ * Returns true if successful.
+ */
+ bool
+ XLogArchiveRevert(const char *xlog)
+ {
+ char archiveStatusPath[MAXPGPATH];
+ FILE *fp;
+ char mark[NUM_PGARCH_SKIP_MARK];
+
+ /*
+ * Reverting a status file might compete with concurrent checkpoint
+ * removing xlog segments. So, WALRemoveLock is required.
+ */
+ LWLockAcquire(WALRemoveLock, LW_EXCLUSIVE);
+
+ StatusFilePath(archiveStatusPath, xlog, ".done");
+ fp = AllocateFile(archiveStatusPath, "r");
+ if (!fp)
+ {
+ if (errno != ENOENT)
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m",
+ archiveStatusPath)));
+ goto not_revert;
+ }
+
+ if (fgets(mark, NUM_PGARCH_SKIP_MARK, fp) == NULL)
+ goto not_revert;
+
+ if (strcmp(mark, PGARCH_SKIP_MARK) != 0)
+ {
+ ereport(WARNING,
+ (errmsg("invalid data in file \"%s\"",
+ archiveStatusPath)));
+ goto not_revert;
+ }
+
+ if (ferror(fp) || FreeFile(fp))
+ {
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("could not read file \"%s\": %m",
+ archiveStatusPath)));
+ goto not_revert;
+ }
+
+ /* Remove the .done file and re-create the .ready */
+ XLogArchiveCleanup(xlog);
+ XLogArchiveNotify(xlog);
+
+ LWLockRelease(WALRemoveLock);
+ return true;
+
+ not_revert:
+ LWLockRelease(WALRemoveLock);
+ return false;
+ }
+
+ /*
* XLogArchiveCheckDone
*
* This is called when we are ready to delete or recycle an old XLOG segment
***************
*** 2758,2763 ****
--- 2828,2835 ----
char lastoff[MAXFNAMELEN];
char path[MAXPGPATH];
+ LWLockAcquire(WALRemoveLock, LW_EXCLUSIVE);
+
/*
* Initialize info about where to try to recycle to. We allow recycling
* segments up to XLOGfileslop segments beyond the current XLOG location.
***************
*** 2767,2776 ****
--- 2839,2851 ----
xldir = AllocateDir(XLOGDIR);
if (xldir == NULL)
+ {
+ LWLockRelease(WALRemoveLock);
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not open transaction log directory \"%s\": %m",
XLOGDIR)));
+ }
XLogFileName(lastoff, ThisTimeLineID, log, seg);
***************
*** 2830,2835 ****
--- 2905,2912 ----
}
FreeDir(xldir);
+
+ LWLockRelease(WALRemoveLock);
}
/*
Index: src/backend/postmaster/pgarch.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/postmaster/pgarch.c,v
retrieving revision 1.38
diff -c -r1.38 pgarch.c
*** src/backend/postmaster/pgarch.c 11 Jan 2008 00:54:08 -0000 1.38
--- src/backend/postmaster/pgarch.c 24 Dec 2008 05:25:01 -0000
***************
*** 40,47 ****
--- 40,50 ----
#include "postmaster/postmaster.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+ #include "storage/lwlock.h"
#include "storage/pg_shmem.h"
#include "storage/pmsignal.h"
+ #include "storage/shmem.h"
+ #include "storage/spin.h"
#include "utils/guc.h"
#include "utils/ps_status.h"
***************
*** 73,78 ****
--- 76,115 ----
/* ----------
+ * Shared data
+ * ----------
+ *
+ * Shared memory area for communication between archiver and other processes.
+ *
+ * This is used to control XLOG archiving dynamically.
+ */
+ typedef struct
+ {
+ /*
+ * skip_pgarch flag indicates whether to skip XLOG archiving. cutoff_xlog
+ * is the criterial file name which is used to determine the log segment
+ * to be skipped.
+ *
+ * If cutoff_xlog is not set, we determine whether to skip XLOG archiving
+ * according to only skip_pgarch. Otherwise, we skip archiving the log
+ * segments
+ * - after or equal to cutoff_xlog if skip_pgarch == true.
+ * - before cutoff_xlog if skip_pgarch == false.
+ */
+ bool skip_pgarch;
+ char cutoff_xlog[MAX_XFN_CHARS + 1];
+
+ slock_t pgarch_lck; /* locks shared variables shown above */
+ } PgArchShmemStruct;
+
+ static PgArchShmemStruct *PgArchShmem;
+
+ #define XLogSkipArchiving(a, c, x) \
+ ((c[0] == '\0' && a) || \
+ ((c[0] != '\0') && (a ? (strcmp(c, x) <= 0) : (strcmp(c, x) > 0))))
+
+
+ /* ----------
* Local data
* ----------
*/
***************
*** 104,111 ****
static void pgarch_MainLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
static bool pgarch_readyXlog(char *xlog);
! static void pgarch_archiveDone(char *xlog);
/* ------------------------------------------------------------
--- 141,150 ----
static void pgarch_MainLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
+ static bool pgarch_skipArchive(char *xlog);
static bool pgarch_readyXlog(char *xlog);
! static void pgarch_archiveDone(char *xlog, bool skipping);
! static bool pgarch_findStatus(char *xlog, char *suffix, bool oldest);
/* ------------------------------------------------------------
***************
*** 167,175 ****
/* Lose the postmaster's on-exit routines */
on_exit_reset();
- /* Drop our connection to postmaster's shared memory, as well */
- PGSharedMemoryDetach();
-
PgArchiverMain(0, NULL);
break;
#endif
--- 206,211 ----
***************
*** 440,449 ****
if (got_SIGTERM || !PostmasterIsAlive(true))
return;
if (pgarch_archiveXlog(xlog))
{
/* successful */
! pgarch_archiveDone(xlog);
break; /* out of inner retry loop */
}
else
--- 476,488 ----
if (got_SIGTERM || !PostmasterIsAlive(true))
return;
+ if (pgarch_skipArchive(xlog))
+ break;
+
if (pgarch_archiveXlog(xlog))
{
/* successful */
! pgarch_archiveDone(xlog, false);
break; /* out of inner retry loop */
}
else
***************
*** 607,612 ****
--- 646,694 ----
}
/*
+ * pgarch_skipArchive
+ *
+ * Determine whether to skip archiving the given xlog file. If yes,
+ * we mark that the xlog file has been successfully archived without
+ * executing archive command, and return true. Otherwise, returns false.
+ */
+ static bool
+ pgarch_skipArchive(char *xlog)
+ {
+ bool skip_pgarch;
+ char cutoff_xlog[MAX_XFN_CHARS + 1];
+ bool ret = false;
+
+ /*
+ * Perform determination and marking atomically for a future
+ * reverting of the notification file.
+ */
+ LWLockAcquire(WALSkipArchLock, LW_EXCLUSIVE);
+
+ /* determine whether to skip archiving according to shmem */
+ {
+ /* use volatile pointer to prevent code rearrangement */
+ volatile PgArchShmemStruct *pgarch = PgArchShmem;
+
+ SpinLockAcquire(&pgarch->pgarch_lck);
+ skip_pgarch = pgarch->skip_pgarch;
+ memcpy(cutoff_xlog, &pgarch->cutoff_xlog, MAX_XFN_CHARS + 1);
+ SpinLockRelease(&pgarch->pgarch_lck);
+ }
+
+ if (XLogSkipArchiving(skip_pgarch, cutoff_xlog, xlog))
+ {
+ ereport(DEBUG1,
+ (errmsg("skip archiving of transaction log file \"%s\"", xlog)));
+ pgarch_archiveDone(xlog, true); /* if we skip archiving, mark so */
+ ret = true;
+ }
+
+ LWLockRelease(WALSkipArchLock);
+ return ret;
+ }
+
+ /*
* pgarch_readyXlog
*
* Return name of the oldest xlog file that has not yet been archived.
***************
*** 630,643 ****
static bool
pgarch_readyXlog(char *xlog)
{
/*
* open xlog status directory and read through list of xlogs that have the
! * .ready suffix, looking for earliest file. It is possible to optimise
! * this code, though only a single file is expected on the vast majority
! * of calls, so....
*/
char XLogArchiveStatusDir[MAXPGPATH];
! char newxlog[MAX_XFN_CHARS + 6 + 1];
DIR *rldir;
struct dirent *rlde;
bool found = false;
--- 712,739 ----
static bool
pgarch_readyXlog(char *xlog)
{
+ return pgarch_findStatus(xlog, ".ready", true);
+ }
+
+ /*
+ * pgarch_findStatus
+ *
+ * Return name of the earliest or latest file which has the given suffix.
+ *
+ * If oldest == true, return name of the earliest file, otherwise latest one.
+ */
+ static bool
+ pgarch_findStatus(char *xlog, char *suffix, bool oldest)
+ {
/*
* open xlog status directory and read through list of xlogs that have the
! * specified suffix, looking for earliest or latest file. It is possible
! * to optimise this code, though only a single file is expected on the vast
! * majority of calls, so....
*/
+ int suffixlen = (int) strlen(suffix);
char XLogArchiveStatusDir[MAXPGPATH];
! char newxlog[MAX_XFN_CHARS + suffixlen + 1];
DIR *rldir;
struct dirent *rlde;
bool found = false;
***************
*** 652,663 ****
while ((rlde = ReadDir(rldir, XLogArchiveStatusDir)) != NULL)
{
! int basenamelen = (int) strlen(rlde->d_name) - 6;
if (basenamelen >= MIN_XFN_CHARS &&
basenamelen <= MAX_XFN_CHARS &&
strspn(rlde->d_name, VALID_XFN_CHARS) >= basenamelen &&
! strcmp(rlde->d_name + basenamelen, ".ready") == 0)
{
if (!found)
{
--- 748,759 ----
while ((rlde = ReadDir(rldir, XLogArchiveStatusDir)) != NULL)
{
! int basenamelen = (int) strlen(rlde->d_name) - suffixlen;
if (basenamelen >= MIN_XFN_CHARS &&
basenamelen <= MAX_XFN_CHARS &&
strspn(rlde->d_name, VALID_XFN_CHARS) >= basenamelen &&
! strcmp(rlde->d_name + basenamelen, suffix) == 0)
{
if (!found)
{
***************
*** 666,672 ****
}
else
{
! if (strcmp(rlde->d_name, newxlog) < 0)
strcpy(newxlog, rlde->d_name);
}
}
--- 762,770 ----
}
else
{
! if (oldest ?
! strcmp(rlde->d_name, newxlog) < 0 :
! strcmp(rlde->d_name, newxlog) > 0)
strcpy(newxlog, rlde->d_name);
}
}
***************
*** 675,682 ****
if (found)
{
! /* truncate off the .ready */
! newxlog[strlen(newxlog) - 6] = '\0';
strcpy(xlog, newxlog);
}
return found;
--- 773,780 ----
if (found)
{
! /* truncate off the suffix */
! newxlog[strlen(newxlog) - suffixlen] = '\0';
strcpy(xlog, newxlog);
}
return found;
***************
*** 689,697 ****
* We do this by renaming the status file from NNN.ready to NNN.done.
* Eventually, a checkpoint process will notice this and delete both the
* NNN.done file and the xlog file itself.
*/
static void
! pgarch_archiveDone(char *xlog)
{
char rlogready[MAXPGPATH];
char rlogdone[MAXPGPATH];
--- 787,799 ----
* We do this by renaming the status file from NNN.ready to NNN.done.
* Eventually, a checkpoint process will notice this and delete both the
* NNN.done file and the xlog file itself.
+ *
+ * If skipping == true, we mark on the NNN.done file that the given xlog
+ * file is not archived. Such a NNN.done file can be reverted to NNN.ready
+ * later, which tries to archive the corresponding xlog file again.
*/
static void
! pgarch_archiveDone(char *xlog, bool skipping)
{
char rlogready[MAXPGPATH];
char rlogdone[MAXPGPATH];
***************
*** 699,706 ****
--- 801,911 ----
StatusFilePath(rlogready, xlog, ".ready");
StatusFilePath(rlogdone, xlog, ".done");
if (rename(rlogready, rlogdone) < 0)
+ {
ereport(WARNING,
(errcode_for_file_access(),
errmsg("could not rename file \"%s\" to \"%s\": %m",
rlogready, rlogdone)));
+ return;
+ }
+
+ /* Mark that archiving this xlog file was skipped */
+ if (skipping)
+ {
+ FILE *fp;
+
+ fp = AllocateFile(rlogdone, "w");
+ if (!fp)
+ {
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m",
+ rlogdone)));
+ return;
+ }
+ fprintf(fp, "%s\n", PGARCH_SKIP_MARK);
+ if (fflush(fp) || ferror(fp) || FreeFile(fp))
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("could not write file \"%s\": %m",
+ rlogdone)));
+ }
+ }
+
+ /*
+ * PgArchShmemSize
+ *
+ * Compute space needed for archiver-related shared memory.
+ */
+ Size
+ PgArchShmemSize(void)
+ {
+ Size size;
+
+ /*
+ * Need the fixed struct
+ */
+ size = MAXALIGN(sizeof(PgArchShmemStruct));
+
+ return size;
+ }
+
+ /*
+ * PgArchShmemInit
+ *
+ * Allocate and initialize archiver-related shared memory.
+ */
+ void
+ PgArchShmemInit(void)
+ {
+ bool found;
+
+ PgArchShmem = (PgArchShmemStruct *)
+ ShmemInitStruct("Archiver Data",
+ PgArchShmemSize(),
+ &found);
+ if (PgArchShmem == NULL)
+ ereport(FATAL,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("not enough shared memory for archiver")));
+ if (found)
+ return; /* already initialized */
+
+ MemSet(PgArchShmem, 0, sizeof(PgArchShmemStruct));
+ SpinLockInit(&PgArchShmem->pgarch_lck);
+ }
+
+ /*
+ * PgArchControl
+ *
+ * Set a flag and a cutoff file name to control XLOG archiving.
+ * If skip_pgarch == true and cutoff_xlog is set, XLOG archiving
+ * begins to be skipped from the log segment which cutoff_xlog
+ * indicates. If skip_pgarch == false and cutoff_xlog is not set,
+ * XLOG archiving resumes immediately from the current log segment.
+ */
+ void
+ PgArchControl(bool skip_pgarch, char *cutoff_xlog)
+ {
+ /* use volatile pointer to prevent code rearrangement */
+ volatile PgArchShmemStruct *pgarch = PgArchShmem;
+
+ SpinLockAcquire(&pgarch->pgarch_lck);
+ pgarch->skip_pgarch = skip_pgarch;
+ if (cutoff_xlog == NULL)
+ MemSet(&pgarch->cutoff_xlog, 0, MAX_XFN_CHARS + 1);
+ else
+ memcpy(&pgarch->cutoff_xlog, cutoff_xlog, MAX_XFN_CHARS + 1);
+ SpinLockRelease(&pgarch->pgarch_lck);
+ }
+
+ /*
+ * GetLastArchivedFileName
+ *
+ * Return name of the latest xlog file which has already been archived.
+ */
+ bool
+ GetLastArchivedFileName(char *xlog)
+ {
+ return pgarch_findStatus(xlog, ".done", false);
}
Index: src/backend/storage/ipc/ipci.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/storage/ipc/ipci.c,v
retrieving revision 1.97
diff -c -r1.97 ipci.c
*** src/backend/storage/ipc/ipci.c 30 Sep 2008 10:52:13 -0000 1.97
--- src/backend/storage/ipc/ipci.c 24 Dec 2008 05:25:03 -0000
***************
*** 24,29 ****
--- 24,30 ----
#include "pgstat.h"
#include "postmaster/autovacuum.h"
#include "postmaster/bgwriter.h"
+ #include "postmaster/pgarch.h"
#include "postmaster/postmaster.h"
#include "storage/bufmgr.h"
#include "storage/ipc.h"
***************
*** 111,116 ****
--- 112,118 ----
size = add_size(size, SInvalShmemSize());
size = add_size(size, BgWriterShmemSize());
size = add_size(size, AutoVacuumShmemSize());
+ size = add_size(size, PgArchShmemSize());
size = add_size(size, BTreeShmemSize());
size = add_size(size, SyncScanShmemSize());
#ifdef EXEC_BACKEND
***************
*** 207,212 ****
--- 209,215 ----
PMSignalInit();
BgWriterShmemInit();
AutoVacuumShmemInit();
+ PgArchShmemInit();
/*
* Set up other modules that need some shared memory space
Index: src/include/access/xlog.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/access/xlog.h,v
retrieving revision 1.88
diff -c -r1.88 xlog.h
*** src/include/access/xlog.h 12 May 2008 08:35:05 -0000 1.88
--- src/include/access/xlog.h 24 Dec 2008 05:25:24 -0000
***************
*** 194,199 ****
--- 194,201 ----
extern void XLogSetAsyncCommitLSN(XLogRecPtr record);
+ extern bool XLogArchiveRevert(const char *xlog);
+
extern void xlog_redo(XLogRecPtr lsn, XLogRecord *record);
extern void xlog_desc(StringInfo buf, uint8 xl_info, char *rec);
Index: src/include/postmaster/pgarch.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/postmaster/pgarch.h,v
retrieving revision 1.6
diff -c -r1.6 pgarch.h
*** src/include/postmaster/pgarch.h 1 Jan 2008 19:45:58 -0000 1.6
--- src/include/postmaster/pgarch.h 24 Dec 2008 05:25:25 -0000
***************
*** 13,18 ****
--- 13,21 ----
#ifndef _PGARCH_H
#define _PGARCH_H
+ #define PGARCH_SKIP_MARK "skip"
+ #define NUM_PGARCH_SKIP_MARK strlen(PGARCH_SKIP_MARK) + 1
+
/* ----------
* Functions called from postmaster
* ----------
***************
*** 23,26 ****
--- 26,38 ----
extern void PgArchiverMain(int argc, char *argv[]);
#endif
+ /* ----------
+ * Functions to control archiving
+ * ----------
+ */
+ extern Size PgArchShmemSize(void);
+ extern void PgArchShmemInit(void);
+ extern void PgArchControl(bool skip_pgarch, char *cutoff_xlog);
+ extern bool GetLastArchivedFileName(char *xlog);
+
#endif /* _PGARCH_H */
Index: src/include/storage/lwlock.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/storage/lwlock.h,v
retrieving revision 1.40
diff -c -r1.40 lwlock.h
*** src/include/storage/lwlock.h 30 Sep 2008 10:52:14 -0000 1.40
--- src/include/storage/lwlock.h 24 Dec 2008 05:25:25 -0000
***************
*** 52,57 ****
--- 52,59 ----
UnusedLock1, /* FreeSpaceMapLock used to be here */
WALInsertLock,
WALWriteLock,
+ WALRemoveLock,
+ WALSkipArchLock,
ControlFileLock,
CheckpointLock,
CLogControlLock,
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
