Here's a cutdown version of the idea about including WAL in the base
backup. What I initially wanted was to introduce a way to guarantee
that the required WAL (with some sort of cutoff of course) would be
available for the backup, but I ran out of time for that. We can
always add that later.
For now, you need to set wal_keep_segments to make it work properly,
but if you do the idea is that the tar file/stream generated in the
base backup will include all the required WAL files. That means that
you can start a postmaster directly against the directory extracted
from the tarball, and you no longer need to set up archive logging to
get a backup.
I've got some refactoring I want to do around the
SendBackupDirectory() function after this, but a review of the
functionality first would be good. And obviously, documentation is
still necessary.
The patch to pg_basebackup applies on top of the previously posted version.
--
Magnus Hagander
Me: http://www.hagander.net/
Work: http://www.redpill-linpro.com/
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index b4d5bbe..d8f0031 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -38,9 +38,9 @@ static void _tarWriteHeader(char *filename, char *linktarget,
struct stat * statbuf);
static void send_int8_string(StringInfoData *buf, int64 intval);
static void SendBackupHeader(List *tablespaces);
-static void SendBackupDirectory(char *location, char *spcoid);
+static void SendBackupDirectory(char *location, char *spcoid, bool closetar);
static void base_backup_cleanup(int code, Datum arg);
-static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir);
+static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool includewal);
typedef struct
{
@@ -67,9 +67,12 @@ base_backup_cleanup(int code, Datum arg)
* clobbered by longjmp" from stupider versions of gcc.
*/
static void
-perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
+perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool includewal)
{
- do_pg_start_backup(backup_label, true);
+ XLogRecPtr startptr;
+ XLogRecPtr endptr;
+
+ startptr = do_pg_start_backup(backup_label, true);
PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
{
@@ -79,11 +82,6 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
tablespaceinfo *ti;
- /* Add a node for the base directory */
- ti = palloc0(sizeof(tablespaceinfo));
- ti->size = progress ? sendDir(".", 1, true) : -1;
- tablespaces = lappend(tablespaces, ti);
-
/* Collect information about all tablespaces */
while ((de = ReadDir(tblspcdir, "pg_tblspc")) != NULL)
{
@@ -111,6 +109,10 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
tablespaces = lappend(tablespaces, ti);
}
+ /* Add a node for the base directory at the end */
+ ti = palloc0(sizeof(tablespaceinfo));
+ ti->size = progress ? sendDir(".", 1, true) : -1;
+ tablespaces = lappend(tablespaces, ti);
/* Send tablespace header */
SendBackupHeader(tablespaces);
@@ -120,12 +122,62 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
{
tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
- SendBackupDirectory(ti->path, ti->oid);
+ SendBackupDirectory(ti->path, ti->oid,
+ !includewal || ti->path != NULL);
}
}
PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
- do_pg_stop_backup();
+ endptr = do_pg_stop_backup();
+
+ if (includewal)
+ {
+ /*
+ * We've left the last tar file "open", so we can now append the
+ * required WAL files to it.
+ */
+ int logid;
+ int logseg;
+
+ elog(LOG, "Going to write wal from %i.%i to %i.%i",
+ startptr.xlogid, startptr.xrecoff / XLogSegSize,
+ endptr.xlogid, endptr.xrecoff / XLogSegSize);
+ logid = startptr.xlogid;
+ logseg = startptr.xrecoff / XLogSegSize;
+
+ while (true)
+ {
+ char xlogname[64];
+ char fn[MAXPGPATH];
+ struct stat statbuf;
+
+ /* Send the current WAL file */
+#define TIMELINE 1
+ XLogFileName(xlogname, TIMELINE, logid, logseg);
+ sprintf(fn, "./pg_xlog/%s", xlogname);
+ if (lstat(fn, &statbuf) != 0)
+ ereport(ERROR,
+ (errcode(errcode_for_file_access()),
+ errmsg("could not stat file \"%s\": %m",
+ fn)));
+
+ if (!S_ISREG(statbuf.st_mode))
+ ereport(ERROR,
+ (errmsg("\"%s\" is not a file",
+ fn)));
+ sendFile(fn, 1, &statbuf);
+
+ /* Advance to the next WAL file */
+ NextLogSeg(logid, logseg);
+
+ /* Have we reached our stop position yet? */
+ if (logid > endptr.xlogid ||
+ (logid == endptr.xlogid && logseg >= endptr.xrecoff / XLogSegSize))
+ break;
+ }
+ /* Send CopyDone message for the last tar file*/
+ pq_putemptymessage('c');
+ }
}
/*
@@ -135,7 +187,7 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
* pg_stop_backup() for the user.
*/
void
-SendBaseBackup(const char *backup_label, bool progress)
+SendBaseBackup(const char *backup_label, bool progress, bool includewal)
{
DIR *dir;
MemoryContext backup_context;
@@ -168,7 +220,7 @@ SendBaseBackup(const char *backup_label, bool progress)
ereport(ERROR,
(errmsg("unable to open directory pg_tblspc: %m")));
- perform_base_backup(backup_label, progress, dir);
+ perform_base_backup(backup_label, progress, dir, includewal);
FreeDir(dir);
@@ -256,7 +308,7 @@ SendBackupHeader(List *tablespaces)
}
static void
-SendBackupDirectory(char *location, char *spcoid)
+SendBackupDirectory(char *location, char *spcoid, bool closetar)
{
StringInfoData buf;
@@ -272,7 +324,8 @@ SendBackupDirectory(char *location, char *spcoid)
false);
/* Send CopyDone message */
- pq_putemptymessage('c');
+ if (closetar)
+ pq_putemptymessage('c');
}
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0ef33dd..d4bbcbe 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -66,11 +66,12 @@ Node *replication_parse_result;
%token K_IDENTIFY_SYSTEM
%token K_LABEL
%token K_PROGRESS
+%token K_WAL
%token K_START_REPLICATION
%type <node> command
%type <node> base_backup start_replication identify_system
-%type <boolval> opt_progress
+%type <boolval> opt_progress opt_wal
%type <str> opt_label
%%
@@ -105,12 +106,13 @@ identify_system:
* BASE_BACKUP [LABEL <label>] [PROGRESS]
*/
base_backup:
- K_BASE_BACKUP opt_label opt_progress
+ K_BASE_BACKUP opt_label opt_progress opt_wal
{
BaseBackupCmd *cmd = (BaseBackupCmd *) makeNode(BaseBackupCmd);
cmd->label = $2;
cmd->progress = $3;
+ cmd->includewal = $4;
$$ = (Node *) cmd;
}
@@ -124,6 +126,10 @@ opt_progress: K_PROGRESS { $$ = true; }
| /* EMPTY */ { $$ = false; }
;
+opt_wal: K_WAL { $$ = true; }
+ | /* EMPTY */ { $$ = false; }
+ ;
+
/*
* START_REPLICATION %X/%X
*/
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index 014a720..9f75346 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -60,6 +60,7 @@ BASE_BACKUP { return K_BASE_BACKUP; }
IDENTIFY_SYSTEM { return K_IDENTIFY_SYSTEM; }
LABEL { return K_LABEL; }
PROGRESS { return K_PROGRESS; }
+WAL { return K_WAL; }
START_REPLICATION { return K_START_REPLICATION; }
"," { return ','; }
";" { return ';'; }
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0ad6804..be3f8ec 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -402,7 +402,7 @@ HandleReplicationCommand(const char *cmd_string)
{
BaseBackupCmd *cmd = (BaseBackupCmd *) cmd_node;
- SendBaseBackup(cmd->label, cmd->progress);
+ SendBaseBackup(cmd->label, cmd->progress, cmd->includewal);
/* Send CommandComplete and ReadyForQuery messages */
EndCommand("SELECT", DestRemote);
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 7fcb20a..eb315b9 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -33,6 +33,7 @@ char *label = "pg_basebackup base backup";
bool showprogress = false;
int verbose = 0;
int compresslevel = 0;
+bool includewal = false;
char *conninfo = NULL;
/* Progress counters */
@@ -97,6 +98,7 @@ usage(void)
printf(_(" -d, --basedir=directory receive base backup into directory\n"));
printf(_(" -t, --tardir=directory receive base backup into tar files\n"
" stored in specified directory\n"));
+ printf(_(" -w, --wal include required WAL files in backup\n"));
printf(_(" -Z, --compress=0-9 compress tar output\n"));
printf(_(" -l, --label=label set backup label\n"));
printf(_(" -p, --progress show progress information\n"));
@@ -642,9 +644,10 @@ BaseBackup()
conn = GetConnection();
PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
- snprintf(current_path, sizeof(current_path), "BASE_BACKUP LABEL '%s' %s",
+ snprintf(current_path, sizeof(current_path), "BASE_BACKUP LABEL '%s' %s %s",
escaped_label,
- showprogress ? "PROGRESS" : "");
+ showprogress ? "PROGRESS" : "",
+ includewal ? "WAL" : "");
if (PQsendQuery(conn, current_path) == 0)
{
@@ -687,7 +690,7 @@ BaseBackup()
* once since it can be relocated, and it will be checked before we do
* anything anyway.
*/
- if (basedir != NULL && i > 0)
+ if (basedir != NULL && !PQgetisnull(res, i, 1))
verify_dir_is_empty_or_create(PQgetvalue(res, i, 1));
}
@@ -747,6 +750,7 @@ main(int argc, char **argv)
{"conninfo", required_argument, NULL, 'c'},
{"basedir", required_argument, NULL, 'd'},
{"tardir", required_argument, NULL, 't'},
+ {"wal", no_argument, NULL, 'w'},
{"compress", required_argument, NULL, 'Z'},
{"label", required_argument, NULL, 'l'},
{"verbose", no_argument, NULL, 'v'},
@@ -776,7 +780,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "c:d:t:l:Z:vp",
+ while ((c = getopt_long(argc, argv, "c:d:t:l:Z:wvp",
long_options, &option_index)) != -1)
{
switch (c)
@@ -790,6 +794,9 @@ main(int argc, char **argv)
case 't':
tardir = xstrdup(optarg);
break;
+ case 'w':
+ includewal = true;
+ break;
case 'l':
label = xstrdup(optarg);
break;
diff --git a/src/include/replication/basebackup.h b/src/include/replication/basebackup.h
index eb2e160..3172d4b 100644
--- a/src/include/replication/basebackup.h
+++ b/src/include/replication/basebackup.h
@@ -12,6 +12,6 @@
#ifndef _BASEBACKUP_H
#define _BASEBACKUP_H
-extern void SendBaseBackup(const char *backup_label, bool progress);
+extern void SendBaseBackup(const char *backup_label, bool progress, bool includewal);
#endif /* _BASEBACKUP_H */
diff --git a/src/include/replication/replnodes.h b/src/include/replication/replnodes.h
index 4f4a1a3..4ecd18a 100644
--- a/src/include/replication/replnodes.h
+++ b/src/include/replication/replnodes.h
@@ -47,6 +47,7 @@ typedef struct BaseBackupCmd
NodeTag type;
char *label;
bool progress;
+ bool includewal;
} BaseBackupCmd;
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
