On Fri, Aug 2, 2019 at 6:43 PM vignesh C <vignes...@gmail.com> wrote:
> Some comments:
> 1) There will be some link files created for tablespace, we might
> require some special handling for it
>
Yep. I have that in my ToDo.
Will start working on that soon.
> 2)
> Retry functionality is hanlded only for copying of full files, should
> we handle retry for copying of partial files
> 3)
> we can have some range for maxretries similar to sleeptime
>
I took help from pg_standby code related to maxentries and sleeptime.
However, as we don't want to use system() call now, I have
removed all this kludge and just used fread/fwrite as discussed.
> 4)
> Should we check for malloc failure
>
Used pg_malloc() instead. Same is also suggested by Ibrar.
>
> 5) Should we add display of progress as backup may take some time,
> this can be added as enhancement. We can get other's opinion on this.
>
Can be done afterward once we have the functionality in place.
>
> 6)
> If the backup count increases providing the input may be difficult,
> Shall user provide all the incremental backups from a parent folder
> and can we handle the ordering of incremental backup internally
>
I am not sure of this yet. We need to provide the tablespace mapping too.
But thanks for putting a point here. Will keep that in mind when I revisit
this.
>
> 7)
> Add verbose for copying whole file
>
Done
>
> 8) We can also check if approximate space is available in disk before
> starting combine backup, this can be added as enhancement. We can get
> other's opinion on this.
>
Hmm... will leave it for now. User will get an error anyway.
>
> 9)
> Combine backup into directory can be combine backup directory
>
Done
>
> 10)
> MAX_INCR_BK_COUNT can be increased little, some applications use 1
> full backup at the beginning of the month and use 30 incremental
> backups rest of the days in the month
>
Yeah, agree. But using any number here is debatable.
Let's see others opinion too.
> Regards,
> Vignesh
> EnterpriseDB: http://www.enterprisedb.com
>
Attached new sets of patches with refactoring done separately.
Incremental backup patch became small now and hopefully more
readable than the first version.
--
Jeevan Chalke
Technical Architect, Product Development
EnterpriseDB Corporation
The Enterprise PostgreSQL Company
From 93041c12a7d07bf17073c9cf4571bd3b5a8acc81 Mon Sep 17 00:00:00 2001
From: Jeevan Chalke <jeevan.cha...@enterprisedb.com>
Date: Fri, 16 Aug 2019 14:08:34 +0530
Subject: [PATCH 1/4] Add support for command line option to pass LSN.
This adds [ LSN 'lsn' ] to BASE_BACKUP command and --lsn=LSN to
the pg_basebackup binary.
Also, add small tests.
---
src/backend/replication/basebackup.c | 20 ++++++++++++++++++++
src/backend/replication/repl_gram.y | 6 ++++++
src/backend/replication/repl_scanner.l | 1 +
src/bin/pg_basebackup/pg_basebackup.c | 15 +++++++++++++--
src/bin/pg_basebackup/t/010_pg_basebackup.pl | 12 +++++++++++-
5 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index c91f66d..74c954b 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -38,6 +38,7 @@
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
+#include "utils/pg_lsn.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/timestamp.h"
@@ -52,6 +53,7 @@ typedef struct
bool includewal;
uint32 maxrate;
bool sendtblspcmapfile;
+ XLogRecPtr lsn;
} basebackup_options;
@@ -638,6 +640,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
bool o_maxrate = false;
bool o_tablespace_map = false;
bool o_noverify_checksums = false;
+ bool o_lsn = false;
MemSet(opt, 0, sizeof(*opt));
foreach(lopt, options)
@@ -726,6 +729,23 @@ parse_basebackup_options(List *options, basebackup_options *opt)
noverify_checksums = true;
o_noverify_checksums = true;
}
+ else if (strcmp(defel->defname, "lsn") == 0)
+ {
+ bool have_error = false;
+
+ if (o_lsn)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("duplicate option \"%s\"", defel->defname)));
+ o_lsn = true;
+
+ /* Validate given LSN and convert it into XLogRecPtr. */
+ opt->lsn = pg_lsn_in_internal(strVal(defel->arg), &have_error);
+ if (XLogRecPtrIsInvalid(opt->lsn))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("invalid value for LSN")));
+ }
else
elog(ERROR, "option \"%s\" not recognized",
defel->defname);
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index c4e11cc..c24d319 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -87,6 +87,7 @@ static SQLCmd *make_sqlcmd(void);
%token K_EXPORT_SNAPSHOT
%token K_NOEXPORT_SNAPSHOT
%token K_USE_SNAPSHOT
+%token K_LSN
%type <node> command
%type <node> base_backup start_replication start_logical_replication
@@ -214,6 +215,11 @@ base_backup_opt:
$$ = makeDefElem("noverify_checksums",
(Node *)makeInteger(true), -1);
}
+ | K_LSN SCONST
+ {
+ $$ = makeDefElem("lsn",
+ (Node *)makeString($2), -1);
+ }
;
create_replication_slot:
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index 380faeb..77b5af4 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -107,6 +107,7 @@ EXPORT_SNAPSHOT { return K_EXPORT_SNAPSHOT; }
NOEXPORT_SNAPSHOT { return K_NOEXPORT_SNAPSHOT; }
USE_SNAPSHOT { return K_USE_SNAPSHOT; }
WAIT { return K_WAIT; }
+LSN { return K_LSN; }
"," { return ','; }
";" { return ';'; }
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 9207109..f8c36ad 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -105,6 +105,7 @@ static bool temp_replication_slot = true;
static bool create_slot = false;
static bool no_slot = false;
static bool verify_checksums = true;
+static char *lsn = NULL;
static bool success = false;
static bool made_new_pgdata = false;
@@ -341,6 +342,7 @@ usage(void)
" include required WAL files with specified method\n"));
printf(_(" -z, --gzip compress tar output\n"));
printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
+ printf(_(" --lsn=LSN incremental backup, using LSN as threshold\n"));
printf(_("\nGeneral options:\n"));
printf(_(" -c, --checkpoint=fast|spread\n"
" set fast or spread checkpointing\n"));
@@ -1801,6 +1803,7 @@ BaseBackup(void)
maxServerMajor;
int serverVersion,
serverMajor;
+ char *lsn_clause = NULL;
Assert(conn != NULL);
@@ -1867,8 +1870,11 @@ BaseBackup(void)
fprintf(stderr, "\n");
}
+ if (lsn)
+ lsn_clause = psprintf("LSN '%s'", lsn);
+
basebkp =
- psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
+ psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s",
escaped_label,
showprogress ? "PROGRESS" : "",
includewal == FETCH_WAL ? "WAL" : "",
@@ -1876,7 +1882,8 @@ BaseBackup(void)
includewal == NO_WAL ? "" : "NOWAIT",
maxrate_clause ? maxrate_clause : "",
format == 't' ? "TABLESPACE_MAP" : "",
- verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
+ verify_checksums ? "" : "NOVERIFY_CHECKSUMS",
+ lsn_clause ? lsn_clause : "");
if (PQsendQuery(conn, basebkp) == 0)
{
@@ -2195,6 +2202,7 @@ main(int argc, char **argv)
{"waldir", required_argument, NULL, 1},
{"no-slot", no_argument, NULL, 2},
{"no-verify-checksums", no_argument, NULL, 3},
+ {"lsn", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
int c;
@@ -2363,6 +2371,9 @@ main(int argc, char **argv)
case 3:
verify_checksums = false;
break;
+ case 4:
+ lsn = pg_strdup(optarg);
+ break;
default:
/*
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index b7d36b6..fd8e187 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -6,7 +6,7 @@ use File::Basename qw(basename dirname);
use File::Path qw(rmtree);
use PostgresNode;
use TestLib;
-use Test::More tests => 106;
+use Test::More tests => 108;
program_help_ok('pg_basebackup');
program_version_ok('pg_basebackup');
@@ -556,5 +556,15 @@ $node->command_ok(
'pg_basebackup with -k does not report checksum mismatch');
rmtree("$tempdir/backup_corrupt4");
+# check LSN
+$node->command_fails(
+ [ 'pg_basebackup', '-D', "$tempdir/lsn_test", '--lsn', "0/INVALID" ],
+ 'pg_basebackup with invalid LSN fails');
+$node->command_ok(
+ [ 'pg_basebackup', '-D', "$tempdir/lsn_test", '--lsn', "0/ABCDEF01", '--no-verify-checksums' ],
+ 'pg_basebackup with valid LSN');
+rmtree("$tempdir/lsn_test");
+
+
$node->safe_psql('postgres', "DROP TABLE corrupt1;");
$node->safe_psql('postgres', "DROP TABLE corrupt2;");
--
1.8.3.1
From ed1d84c3ac81c493ed0f4812b292a1d82f858936 Mon Sep 17 00:00:00 2001
From: Jeevan Chalke <jeevan.cha...@enterprisedb.com>
Date: Fri, 16 Aug 2019 14:11:08 +0530
Subject: [PATCH 4/4] Add support to combine files using pg_combinebackup.
---
doc/src/sgml/ref/allfiles.sgml | 1 +
doc/src/sgml/ref/pg_basebackup.sgml | 2 +-
doc/src/sgml/ref/pg_combinebackup.sgml | 181 ++++++
doc/src/sgml/reference.sgml | 1 +
src/bin/Makefile | 1 +
src/bin/pg_combinebackup/Makefile | 42 ++
src/bin/pg_combinebackup/pg_combinebackup.c | 916 ++++++++++++++++++++++++++++
7 files changed, 1143 insertions(+), 1 deletion(-)
create mode 100644 doc/src/sgml/ref/pg_combinebackup.sgml
create mode 100644 src/bin/pg_combinebackup/Makefile
create mode 100644 src/bin/pg_combinebackup/pg_combinebackup.c
diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml
index 8d91f35..f3e90b6 100644
--- a/doc/src/sgml/ref/allfiles.sgml
+++ b/doc/src/sgml/ref/allfiles.sgml
@@ -200,6 +200,7 @@ Complete list of usable sgml source files in this directory.
<!ENTITY pgBasebackup SYSTEM "pg_basebackup.sgml">
<!ENTITY pgbench SYSTEM "pgbench.sgml">
<!ENTITY pgChecksums SYSTEM "pg_checksums.sgml">
+<!ENTITY pgCombinebackup SYSTEM "pg_combinebackup.sgml">
<!ENTITY pgConfig SYSTEM "pg_config-ref.sgml">
<!ENTITY pgControldata SYSTEM "pg_controldata.sgml">
<!ENTITY pgCtl SYSTEM "pg_ctl-ref.sgml">
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 00782e0..92d9d13 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -415,7 +415,7 @@ PostgreSQL documentation
which are modified after this given LSN will be backed up. The file
which has these partial blocks has .partial as an extension. Backup
taken in this manner has to be combined with the full backup with the
- <command>pg_combinebackup</command> utility.
+ <xref linkend="app-pgcombinebackup"/> utility.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/pg_combinebackup.sgml b/doc/src/sgml/ref/pg_combinebackup.sgml
new file mode 100644
index 0000000..5aefbf8
--- /dev/null
+++ b/doc/src/sgml/ref/pg_combinebackup.sgml
@@ -0,0 +1,181 @@
+<!--
+doc/src/sgml/ref/pg_combinebackup.sgml
+PostgreSQL documentation
+-->
+
+<refentry id="app-pgcombinebackup">
+ <indexterm zone="app-pgcombinebackup">
+ <primary>pg_combinebackup</primary>
+ </indexterm>
+
+ <refmeta>
+ <refentrytitle><application>pg_combinebackup</application></refentrytitle>
+ <manvolnum>1</manvolnum>
+ <refmiscinfo>Application</refmiscinfo>
+ </refmeta>
+
+ <refnamediv>
+ <refname>pg_combinebackup</refname>
+ <refpurpose>create a synthetic backup from a full backup and one or more incremental backups</refpurpose>
+ </refnamediv>
+
+ <refsynopsisdiv>
+ <cmdsynopsis>
+ <command>pg_combinebackup</command>
+ <arg rep="repeat" choice="opt"><replaceable class="parameter">option</replaceable></arg>
+ </cmdsynopsis>
+ </refsynopsisdiv>
+
+ <refsect1 id="r1-app-pg_combinebackup-1">
+ <title>Description</title>
+ <para>
+ <application>pg_combinebackup</application> combines one or more incremental
+ backups with the full base-backup to generate a synthetic backup.
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>Options</title>
+
+ <para>
+ The following command-line options are available:
+
+ <variablelist>
+ <varlistentry>
+ <term><option>-f <replaceable>directory</replaceable></option></term>
+ <term><option>--full-backup=<replaceable>directory</replaceable></option></term>
+ <listitem>
+ <para>
+ Specifies the directory where the full backup is stored.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-i <replaceable>directory</replaceable></option></term>
+ <term><option>--incr-backup=<replaceable>directory</replaceable></option></term>
+ <listitem>
+ <para>
+ Specifies the directory where the incremental backup is stored.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-o <replaceable>directory</replaceable></option></term>
+ <term><option>--output-backup=<replaceable>directory</replaceable></option></term>
+ <listitem>
+ <para>
+ Specifies the output directory where the combined full synthetic backup
+ to be stored.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-n</option></term>
+ <term><option>--no-clean</option></term>
+ <listitem>
+ <para>
+ By default, when <command>pg_combinebackup</command> aborts with an
+ error, it removes the output data directories it might have created
+ before discovering that it cannot finish the job. This option inhibits
+ tidying-up and is thus useful for debugging.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-v</option></term>
+ <term><option>--verbose</option></term>
+ <listitem>
+ <para>
+ Enable verbose output. Lists all partial files processed and its
+ checksum status.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-V</option></term>
+ <term><option>--version</option></term>
+ <listitem>
+ <para>
+ Print the <application>pg_combinebackup</application> version and exit.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>-?</option></term>
+ <term><option>--help</option></term>
+ <listitem>
+ <para>
+ Show help about <application>pg_combinebackup</application> command line
+ arguments, and exit.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>Environment</title>
+ <variablelist>
+ <varlistentry>
+ <term><envar>PG_COLOR</envar></term>
+ <listitem>
+ <para>
+ Specifies whether to use color in diagnostics messages. Possible values
+ are <literal>always</literal>, <literal>auto</literal>,
+ <literal>never</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </refsect1>
+
+ <refsect1>
+ <title>Notes</title>
+ <para>
+ Output directory, full backup directory, and at-least one incremental backup
+ directory must be specified.
+ </para>
+
+ <para>
+ <literal>PREVIOUS WAL LOCATION</literal> of the incremental backup must
+ match with the <literal>START WAL LOCATION</literal> of the previous full
+ or incremental backup in a given sequence.
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>Examples</title>
+
+ <para>
+ To combine a full backup with two incremental backups and store it in the
+ output directory:
+<screen>
+<prompt>$</prompt> <userinput>pg_combinebackup -f /data/full/data -i /data/incr/data1 -i /data/incr/data2 -o /data/full/fulldata</userinput>
+</screen>
+ </para>
+
+ <para>
+ To combine a full backup with an incremental backups and store it in the
+ output directory along with various options like, verbose, no-clean etc.:
+<screen>
+<prompt>$</prompt> <userinput>pg_combinebackup -v --no-clean -f /data/full/data -i /data/incr/data1 -o /data/full/fulldata</userinput>
+</screen>
+ </para>
+ </refsect1>
+
+ <refsect1>
+ <title>See Also</title>
+
+ <simplelist type="inline">
+ <member><xref linkend="app-pgbasebackup"/></member>
+ </simplelist>
+ </refsect1>
+
+</refentry>
diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml
index cef09dd..3513ab4 100644
--- a/doc/src/sgml/reference.sgml
+++ b/doc/src/sgml/reference.sgml
@@ -248,6 +248,7 @@
&ecpgRef;
&pgBasebackup;
&pgbench;
+ &pgCombinebackup;
&pgConfig;
&pgDump;
&pgDumpall;
diff --git a/src/bin/Makefile b/src/bin/Makefile
index 903e581..fc3cea4 100644
--- a/src/bin/Makefile
+++ b/src/bin/Makefile
@@ -18,6 +18,7 @@ SUBDIRS = \
pg_archivecleanup \
pg_basebackup \
pg_checksums \
+ pg_combinebackup \
pg_config \
pg_controldata \
pg_ctl \
diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile
new file mode 100644
index 0000000..bdc9219
--- /dev/null
+++ b/src/bin/pg_combinebackup/Makefile
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/bin/pg_combinebackup
+#
+# Copyright (c) 1998-2019, PostgreSQL Global Development Group
+#
+# src/bin/pg_combinebackup/Makefile
+#
+#-------------------------------------------------------------------------
+
+PGFILEDESC = "pg_combinebackup - combine full backup with incremental backups"
+PGAPPICON=win32
+
+subdir = src/bin/pg_combinebackup
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS= pg_combinebackup.o $(WIN32RES)
+
+all: pg_combinebackup
+
+pg_combinebackup: $(OBJS) | submake-libpgport
+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+install: all installdirs
+ $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)'
+
+installdirs:
+ $(MKDIR_P) '$(DESTDIR)$(bindir)'
+
+uninstall:
+ rm -f '$(DESTDIR)$(bindir)/pg_combinebackup$(X)'
+
+clean distclean maintainer-clean:
+ rm -f pg_combinebackup$(X) $(OBJS)
+ rm -rf tmp_check
+
+check:
+ $(prove_check)
+
+installcheck:
+ $(prove_installcheck)
diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c
new file mode 100644
index 0000000..2f5447f
--- /dev/null
+++ b/src/bin/pg_combinebackup/pg_combinebackup.c
@@ -0,0 +1,916 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_combinebackup.c
+ * Combines one or more incremental backups with the full base-backup to
+ * generate new full base-backup.
+ *
+ * Copyright (c) 2010-2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/bin/pg_combinebackup/pg_combinebackup.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include <dirent.h>
+#include <time.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/xlog_internal.h"
+#include "common/controldata_utils.h"
+#include "common/file_perm.h"
+#include "common/logging.h"
+#include "getopt_long.h"
+#include "pg_getopt.h"
+#include "replication/basebackup.h"
+
+
+/* Max number of incremental backups to be combined. */
+#define MAX_INCR_BK_COUNT 10
+
+
+typedef struct
+{
+ FILE *fp;
+ char filename[MAXPGPATH];
+ bool isPartial;
+} FileMap;
+
+typedef struct
+{
+ FILE *fp;
+ int offset;
+} FileOffset;
+
+static const char *progname;
+static ControlFileData *ControlFile;
+static bool verbose = false;
+static bool success = false;
+static bool noclean = false;
+static bool made_new_outputdata = false;
+static bool found_existing_outputdata = false;
+static bool checksum_failure = false;
+static char *OutputDir = NULL;
+
+/* Function headers */
+static void usage(void);
+static void check_compatibility(char *datadir);
+static void verify_dir_is_empty_or_create(char *dirname, bool *created,
+ bool *found);
+static void cleanup_directories_atexit(void);
+static void combine_partial_files(const char *fn, char **IncrDirs,
+ int nIncrDir, const char *subdirpath,
+ const char*outfn);
+static void copy_whole_file(const char *fromfn, const char *tofn);
+static void cleanup_filemaps(FileMap *filemaps, int nfilemaps);
+
+
+static void
+usage(void)
+{
+ printf(_("%s combines full backup with incremental backup.\n\n"), progname);
+ printf(_("Usage:\n"));
+ printf(_(" %s [OPTION]...\n"), progname);
+ printf(_("\nOptions:\n"));
+ printf(_(" -f, --full-backup=DIRECTORY full backup directory\n"));
+ printf(_(" -i, --incr-backup=DIRECTORY incremental backup directory (maximum %d)\n"), MAX_INCR_BK_COUNT);
+ printf(_(" -o, --output-dir=DIRECTORY combine backup directory\n"));
+ printf(_("\nGeneral options:\n"));
+ printf(_(" -n, --no-clean do not clean up after errors\n"));
+ printf(_(" -v, --verbose output verbose messages\n"));
+ printf(_(" -V, --version output version information, then exit\n"));
+ printf(_(" -?, --help show this help, then exit\n"));
+ printf(_("\nReport bugs to <pgsql-b...@lists.postgresql.org>.\n"));
+}
+
+/*
+ * scan_file
+ *
+ * Checks whether given file is partial file or not. If partial, then combines
+ * it into a full backup file, else copies as is to the output directory.
+ */
+static void
+scan_file(const char *fn, char **IncrDirs, int nIncrDir,
+ const char *subdirpath)
+{
+ char *extptr = strstr(fn, ".partial");
+
+ /* If .partial file, combine them, else copy it as is */
+ if (extptr != NULL)
+ {
+ char outfn[MAXPGPATH];
+
+ if (verbose)
+ pg_log_info("combining partial file \"%s\"", fn);
+
+ if (subdirpath)
+ snprintf(outfn, MAXPGPATH, "%s/%s/%s", OutputDir, subdirpath, fn);
+ else
+ snprintf(outfn, MAXPGPATH, "%s/%s", OutputDir, fn);
+
+ extptr = strstr(outfn, ".partial");
+ Assert (extptr != NULL);
+ extptr[0] = '\0';
+
+ combine_partial_files(fn, IncrDirs, nIncrDir, subdirpath, outfn);
+ }
+ else
+ {
+ char infn[MAXPGPATH];
+ char outfn[MAXPGPATH];
+
+ if (verbose)
+ pg_log_info("copying file \"%s\"", fn);
+
+ if (subdirpath)
+ {
+ snprintf(infn, MAXPGPATH, "%s/%s/%s", IncrDirs[nIncrDir - 1],
+ subdirpath, fn);
+ snprintf(outfn, MAXPGPATH, "%s/%s/%s", OutputDir, subdirpath, fn);
+ }
+ else
+ {
+ snprintf(infn, MAXPGPATH, "%s/%s", IncrDirs[nIncrDir - 1], fn);
+ snprintf(outfn, MAXPGPATH, "%s/%s", OutputDir, fn);
+ }
+
+ copy_whole_file(infn, outfn);
+ }
+}
+
+/*
+ * copy_whole_file
+ *
+ * Copy file from source to its destination.
+ */
+static void
+copy_whole_file(const char *fromfn, const char *tofn)
+{
+ FILE *ifp;
+ FILE *ofp;
+ char *buf;
+ struct stat statbuf;
+ off_t cnt;
+ pgoff_t len = 0;
+
+ ifp = fopen(fromfn, "rb");
+ if (ifp == NULL)
+ {
+ pg_log_error("could not open file \"%s\": %m", fromfn);
+ exit(1);
+ }
+
+ if (fstat(fileno(ifp), &statbuf) != 0)
+ {
+ pg_log_error("could not stat file \"%s\": %m", fromfn);
+ fclose(ifp);
+ exit(1);
+ }
+
+ if (verbose && statbuf.st_size > (RELSEG_SIZE * BLCKSZ))
+ pg_log_info("found big file \"%s\" (size: %.2lfGB): %m", fromfn,
+ (double) statbuf.st_size / (RELSEG_SIZE * BLCKSZ));
+
+ ofp = fopen(tofn, "wb");
+ if (ofp == NULL)
+ {
+ pg_log_error("could not create file \"%s\": %m", tofn);
+ exit(1);
+ }
+
+ /* 1GB slice */
+ buf = (char *) pg_malloc(RELSEG_SIZE * BLCKSZ);
+
+ /*
+ * We do read entire 1GB file in memory while taking incremental backup; so
+ * I don't see any reason why can't we do that here. Also, copying data in
+ * chunks is expensive. However, for bigger files, we still slice at 1GB
+ * border.
+ */
+ while ((cnt = fread(buf, 1, Min(RELSEG_SIZE * BLCKSZ, statbuf.st_size - len), ifp)) > 0)
+ {
+ /* Write the buf to the output file. */
+ if (fwrite(buf, 1, cnt, ofp) != cnt)
+ {
+ pg_log_error("could not write to file \"%s\": %m", tofn);
+ fclose(ifp);
+ fclose(ofp);
+ pg_free(buf);
+ exit(1);
+ }
+
+ len += cnt;
+ }
+
+ if (len < statbuf.st_size)
+ pg_log_error("could not read file \"%s\": %m", fromfn);
+
+ fclose(ifp);
+ fclose(ofp);
+ pg_free(buf);
+}
+
+/*
+ * scan_directory
+ *
+ * Scan the input incremental directory and operates on each file. Creates
+ * corresponding directories in the output directory too.
+ */
+static void
+scan_directory(char **IncrDirs, int nIncrDir, const char *subdirpath)
+{
+ char path[MAXPGPATH];
+ DIR *dir;
+ struct dirent *de;
+
+ if (subdirpath)
+ {
+ char outputpath[MAXPGPATH];
+
+ snprintf(path, sizeof(path), "%s/%s", IncrDirs[nIncrDir - 1],
+ subdirpath);
+ snprintf(outputpath, sizeof(outputpath), "%s/%s", OutputDir,
+ subdirpath);
+
+ /* Create this sub-directory in output directory */
+ if (pg_mkdir_p(outputpath, pg_dir_create_mode) == -1)
+ {
+ pg_log_error("could not create directory \"%s\": %m", outputpath);
+ exit(1);
+ }
+ }
+ else
+ snprintf(path, sizeof(path), "%s", IncrDirs[nIncrDir - 1]);
+
+ dir = opendir(path);
+ if (!dir)
+ {
+ pg_log_error("could not open directory \"%s\": %m", path);
+ exit(1);
+ }
+
+ while ((de = readdir(dir)) != NULL)
+ {
+ char fn[MAXPGPATH];
+ struct stat st;
+
+ if (strcmp(de->d_name, ".") == 0 ||
+ strcmp(de->d_name, "..") == 0)
+ continue;
+
+ snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
+ if (lstat(fn, &st) < 0)
+ {
+ pg_log_error("could not stat file \"%s\": %m", fn);
+ exit(1);
+ }
+ if (S_ISREG(st.st_mode))
+ scan_file(de->d_name, IncrDirs, nIncrDir, subdirpath);
+#ifndef WIN32
+ else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
+#else
+ else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
+#endif
+ {
+ char newsubdirpath[MAXPGPATH];
+
+ if (subdirpath)
+ snprintf(newsubdirpath, MAXPGPATH, "%s/%s", subdirpath,
+ de->d_name);
+ else
+ snprintf(newsubdirpath, MAXPGPATH, "%s", de->d_name);
+
+ scan_directory(IncrDirs, nIncrDir, newsubdirpath);
+ }
+ }
+ closedir(dir);
+ return;
+}
+
+int
+main(int argc, char *argv[])
+{
+ static struct option long_options[] = {
+ {"full-backup", required_argument, NULL, 'f'},
+ {"incr-backup", required_argument, NULL, 'i'},
+ {"output-dir", required_argument, NULL, 'o'},
+ {"no-clean", no_argument, NULL, 'n'},
+ {"verbose", no_argument, NULL, 'v'},
+ {NULL, 0, NULL, 0}
+ };
+
+ char *IncrDirs[MAX_INCR_BK_COUNT];
+ int nIncrDir;
+ int c;
+ int option_index;
+ int i;
+ XLogRecPtr startlsn = InvalidXLogRecPtr;
+ XLogRecPtr prevlsn = InvalidXLogRecPtr;
+
+ pg_logging_init(argv[0]);
+ set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
+ progname = get_progname(argv[0]);
+
+ if (argc > 1)
+ {
+ if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
+ {
+ usage();
+ exit(0);
+ }
+ if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
+ {
+ puts("pg_combinebackup (PostgreSQL) " PG_VERSION);
+ exit(0);
+ }
+ }
+
+ atexit(cleanup_directories_atexit);
+
+ /* Zero index is reserved for full backup directory. */
+ IncrDirs[0] = NULL;
+ nIncrDir = 1;
+ while ((c = getopt_long(argc, argv, "f:ni:o:v", long_options, &option_index)) != -1)
+ {
+ switch (c)
+ {
+ case 'f':
+ IncrDirs[0] = optarg;
+ break;
+ case 'n':
+ noclean = true;
+ break;
+ case 'i':
+ if (nIncrDir == MAX_INCR_BK_COUNT)
+ {
+ pg_log_error("too many incremental backups to combine");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
+ exit(1);
+ }
+
+ IncrDirs[nIncrDir] = optarg;
+ nIncrDir++;
+ break;
+ case 'o':
+ OutputDir = optarg;
+ break;
+ case 'v':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
+ exit(1);
+ }
+ }
+
+ /*
+ * Need to have directory paths for full backup, incremental backups, and
+ * the output directory. Error out if we don't get that.
+ */
+ if (IncrDirs[0] == NULL)
+ {
+ pg_log_error("no full backup directory specified");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+ if (nIncrDir == 1)
+ {
+ pg_log_error("no incremental backup directory specified");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+ if (OutputDir == NULL)
+ {
+ pg_log_error("no target directory specified");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+ else
+ verify_dir_is_empty_or_create(OutputDir, &made_new_outputdata,
+ &found_existing_outputdata);
+
+ /* Complain if any arguments remain */
+ if (optind < argc)
+ {
+ pg_log_error("too many command-line arguments (first is \"%s\")",
+ argv[optind]);
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+ progname);
+ exit(1);
+ }
+
+ /*
+ * Verify the backup chain. INCREMENTAL BACKUP REFERENCE WAL LOCATION of
+ * the incremental backup must match with the START WAL LOCATION of the
+ * previous backup, until we reach a full backup in which there is no
+ * INCREMENTAL BACKUP REFERENCE WAL LOCATION.
+ */
+ for (i = (nIncrDir - 1); i >= 0; i--)
+ {
+ struct stat statbuf;
+ char filename[MAXPGPATH];
+ FILE *fp;
+ char *labelfile;
+ char startxlogfilename[MAXFNAMELEN];
+ uint32 hi;
+ uint32 lo;
+ char ch;
+
+ /*
+ * BACKUP_LABEL_FILE is defined in xlog.h which needs postgres.h to be
+ * included too. Thus to avoid that define it here again.
+ */
+#define BACKUP_LABEL_FILE "backup_label"
+
+ check_compatibility(IncrDirs[i]);
+
+ snprintf(filename, MAXPGPATH, "%s/%s", IncrDirs[i], BACKUP_LABEL_FILE);
+ if (stat(filename, &statbuf))
+ {
+ pg_log_error("could not stat file \"%s\": %m", filename);
+ exit(1);
+ }
+ fp = fopen(filename, "r");
+ if (fp == NULL)
+ {
+ pg_log_error("could not read file \"%s\": %m", filename);
+ exit(1);
+ }
+
+ labelfile = pg_malloc(statbuf.st_size + 1);
+ if (fread(labelfile, 1, statbuf.st_size, fp) != statbuf.st_size)
+ {
+ pg_log_error("corrupted file \"%s\": %m", filename);
+ pg_free(labelfile);
+ exit(1);
+ }
+
+ labelfile[statbuf.st_size] = '\0';
+
+ /*
+ * Read the START WAL LOCATION from the directory, we skip this for top
+ * most directory corresponding to the last incremental backup as it is
+ * not needed to check.
+ */
+ if (i != (nIncrDir - 1))
+ {
+ if (sscanf(labelfile, "START WAL LOCATION: %X/%X (file %24s)%c",
+ &hi, &lo, startxlogfilename,
+ &ch) != 4 || ch != '\n')
+ {
+ pg_log_error("invalid data in file \"%s\": %m", filename);
+ pg_free(labelfile);
+ exit(1);
+ }
+ startlsn = ((uint64) hi) << 32 | lo;
+
+ /*
+ * We end up here from second loop counter, thus prevlsn must have
+ * been already set. Check that with startlsn fetched above, they
+ * must match. Otherwise we have a broken chain, bail out.
+ */
+ Assert(prevlsn != InvalidXLogRecPtr);
+ if (prevlsn != startlsn)
+ {
+ pg_log_error("invalid backup chain");
+ pg_free(labelfile);
+ exit(1);
+ }
+ }
+
+ /*
+ * Fetch the INCREMENTAL BACKUP REFERENCE WAL LOCATION from the
+ * incremental backup directory. Index 0 is of full backup directory
+ * where we won't have that, so we skip it.
+ */
+ if (i != 0)
+ {
+ char *ptr = strstr(labelfile, "INCREMENTAL BACKUP REFERENCE WAL LOCATION:");
+
+ if (!ptr || sscanf(ptr, "INCREMENTAL BACKUP REFERENCE WAL LOCATION: %X/%X", &hi, &lo) != 2)
+ {
+ pg_log_error("invalid data in file \"%s\": %m", filename);
+ pg_free(labelfile);
+ exit(1);
+ }
+ prevlsn = ((uint64) hi) << 32 | lo;
+ }
+
+ pg_free(labelfile);
+ fclose(fp);
+ }
+
+ /* Scan whole directory and process all .partial files */
+ scan_directory(IncrDirs, nIncrDir, NULL);
+
+ success = true;
+ return 0;
+}
+
+/*
+ * check_compatibility
+ *
+ * Read the control file and check compatibility
+ */
+static void
+check_compatibility(char *datadir)
+{
+ bool crc_ok;
+
+ ControlFile = get_controlfile(datadir, &crc_ok);
+ if (!crc_ok)
+ {
+ pg_log_error("pg_control CRC value is incorrect");
+ exit(1);
+ }
+
+ if (ControlFile->pg_control_version != PG_CONTROL_VERSION)
+ {
+ pg_log_error("cluster is not compatible with this version of pg_combinebackup");
+ exit(1);
+ }
+
+ if (ControlFile->blcksz != BLCKSZ)
+ {
+ pg_log_error("database cluster is not compatible");
+ fprintf(stderr, _("The database cluster was initialized with block size %u, but pg_combinebackup was compiled with block size %u.\n"),
+ ControlFile->blcksz, BLCKSZ);
+ exit(1);
+ }
+
+ /* When backup was taken, the database should have been in clean state. */
+ if (ControlFile->state != DB_IN_PRODUCTION)
+ {
+ pg_log_error("cluster must be in production");
+ exit(1);
+ }
+}
+
+/*
+ * verify_dir_is_empty_or_create
+ *
+ * Verify that the given directory exists and is empty. If it does not exists,
+ * it is created. If it exists but is not empty, an error will be given and
+ * the process ended.
+ */
+static void
+verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
+{
+ switch (pg_check_dir(dirname))
+ {
+ case 0:
+ /*
+ * Does not exist, so create
+ */
+ if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
+ {
+ pg_log_error("could not create directory \"%s\": %m", dirname);
+ exit(1);
+ }
+ if (created)
+ *created = true;
+ return;
+
+ case 1:
+ /*
+ * Exists, empty
+ */
+ if (found)
+ *found = true;
+ return;
+
+ case 2:
+ case 3:
+ case 4:
+ /*
+ * Exists, not empty
+ */
+ pg_log_error("directory \"%s\" exists but is not empty", dirname);
+ exit(1);
+
+ case -1:
+ /*
+ * Access problem
+ */
+ pg_log_error("could not access directory \"%s\": %m", dirname);
+ exit(1);
+ }
+}
+
+static void
+cleanup_directories_atexit(void)
+{
+ if (success)
+ return;
+
+ if (!noclean && !checksum_failure)
+ {
+ if (made_new_outputdata)
+ {
+ pg_log_info("removing target data directory \"%s\"", OutputDir);
+ if (!rmtree(OutputDir, true))
+ pg_log_error("failed to remove data directory");
+ }
+ else if (found_existing_outputdata)
+ {
+ pg_log_info("removing contents of target data directory \"%s\"",
+ OutputDir);
+ if (!rmtree(OutputDir, false))
+ pg_log_error("failed to remove contents of data directory");
+ }
+ }
+ else
+ {
+ if ((made_new_outputdata || found_existing_outputdata) &&
+ !checksum_failure)
+ pg_log_info("target data directory \"%s\" not removed at user's request",
+ OutputDir);
+ }
+}
+
+/*
+ * combine_partial_files
+ *
+ * Combines one or more incremental backups with full backup. The algorithm in
+ * this function works this way:
+ * 1. Work backward through the backup chain until we find a complete version
+ * of the file. We create a filemap in this process.
+ * 2. Loop over all the files within filemap, read the header and check the
+ * blocks modified, verify the CRC and create a blockmap.
+ * 3. Create a new file in output directory by writing all the blocks.
+ */
+static void
+combine_partial_files(const char *fn, char **IncrDirs, int nIncrDir,
+ const char *subdirpath, const char *outfn)
+{
+ FILE *outfp;
+ FileOffset outblocks[RELSEG_SIZE];
+ int i;
+ FileMap *filemaps;
+ int fmindex;
+ bool basefilefound;
+ bool modifiedblockfound;
+ uint32 lastblkno;
+ FileMap *fm;
+ struct stat statbuf;
+ uint32 nblocks;
+
+ memset(outblocks, 0, sizeof(FileOffset) * RELSEG_SIZE);
+ filemaps = (FileMap *) pg_malloc(sizeof(FileMap) * nIncrDir);
+
+ /*
+ * Open all files from all incremental backup directories and create a file
+ * map.
+ */
+ basefilefound = false;
+ for (i = (nIncrDir - 1), fmindex = 0; i >= 0; i--, fmindex++)
+ {
+ fm = &filemaps[fmindex];
+
+ if (subdirpath)
+ snprintf(fm->filename, MAXPGPATH, "%s/%s/%s", IncrDirs[i],
+ subdirpath, fn);
+ else
+ snprintf(fm->filename, MAXPGPATH, "%s/%s", IncrDirs[i], fn);
+
+ fm->fp = fopen(fm->filename, "rb");
+ if (fm->fp == NULL)
+ {
+ if (errno == ENOENT)
+ {
+ char *extptr = strstr(fm->filename, ".partial");
+
+ Assert (extptr != NULL);
+ extptr[0] = '\0';
+
+ /* Check without .partial */
+ fm->fp = fopen(fm->filename, "rb");
+ if (fm->fp != NULL)
+ {
+ fm->isPartial = false;
+ basefilefound = true;
+ /* We got a non-partial file, so no need to scan further */
+ break;
+ }
+ }
+
+ pg_log_error("could not open file \"%s\": %m", fm->filename);
+ pg_free(filemaps);
+ exit(1);
+ }
+ else
+ {
+ fm->isPartial = true;
+ }
+ }
+
+ /* We must have found the base file. */
+ if (!basefilefound)
+ {
+ pg_log_error("could not find base file \"%s\": %m", fn);
+ pg_free(filemaps);
+ exit(1);
+ }
+
+ /* Process all opened files. */
+ lastblkno = 0;
+ modifiedblockfound = false;
+ for (i = 0; i < fmindex; i++)
+ {
+ char *buf;
+ int hsize;
+ int k;
+ int blkstartoffset;
+ int blknumberssize;
+ uint32 *blknumbers;
+ partial_file_header *pfh;
+ pg_crc32c savedchecksum;
+
+ fm = &filemaps[i];
+ Assert(fm->isPartial);
+
+ hsize = offsetof(partial_file_header, blocknumbers);
+ buf = (char *) pg_malloc(hsize);
+
+ /* Read partial file header. */
+ if (fread(buf, 1, hsize, fm->fp) != hsize)
+ {
+ pg_log_error("corrupted partial file \"%s\": %m", fm->filename);
+ pg_free(filemaps);
+ pg_free(buf);
+ exit(1);
+ }
+
+ pfh = (partial_file_header *) buf;
+
+ /* Check magic */
+ if (pfh->magic != INCREMENTAL_BACKUP_MAGIC)
+ {
+ pg_log_error("corrupted partial file \"%s\", magic mismatch: %m", fm->filename);
+ pg_free(filemaps);
+ pg_free(buf);
+ exit(1);
+ }
+
+ blknumberssize = sizeof(uint32) * pfh->nblocks;
+ blknumbers = (uint32 *) pg_malloc(blknumberssize);
+
+ /* Read all block numbers. */
+ if (fread((char *) blknumbers, 1, blknumberssize, fm->fp) != blknumberssize)
+ {
+ pg_log_error("corrupted partial file \"%s\": %m", fm->filename);
+ pg_free(blknumbers);
+ pg_free(buf);
+ pg_free(filemaps);
+ exit(1);
+ }
+
+ /* Check CRC */
+ savedchecksum = pfh->checksum;
+ INIT_CRC32C(pfh->checksum);
+ COMP_CRC32C(pfh->checksum, pfh, hsize);
+ COMP_CRC32C(pfh->checksum, blknumbers, blknumberssize);
+ if (pfh->checksum != savedchecksum)
+ {
+ pg_log_error("corrupted partial file \"%s\", checksum mismatch: %m", fm->filename);
+ pg_free(blknumbers);
+ pg_free(filemaps);
+ pg_free(buf);
+ exit(1);
+ }
+ else if (verbose)
+ pg_log_info("checksums verified in file \"%s\"", fm->filename);
+
+ blkstartoffset = hsize + blknumberssize;
+ for (k = 0; k < pfh->nblocks; k++)
+ {
+ uint32 blknum = blknumbers[k];
+
+ /*
+ * Set this block pointer in outblock array. We skip setting
+ * it if already set as we are processing from latest file to
+ * oldest file. If same block is modified across multiple
+ * incremental backup, then we use the latest one; skipping all
+ * other.
+ */
+ if (outblocks[blknum].fp == NULL)
+ {
+ outblocks[blknum].fp = fm->fp;
+ outblocks[blknum].offset = blkstartoffset + BLCKSZ * k;
+ }
+
+ modifiedblockfound = true;
+ }
+
+ /* Update last block number */
+ if (k != 0 && blknumbers[k - 1] > lastblkno)
+ lastblkno = (int) blknumbers[k - 1];
+ }
+
+ /* Read base file */
+ Assert(i == fmindex);
+
+ fm = &filemaps[fmindex];
+ Assert(fm->isPartial == false);
+
+ /*
+ * If after processing all .partial files, we end up with no blocks
+ * modified, then simply copy the base file to the output directory and
+ * we are done.
+ */
+ if (!modifiedblockfound)
+ {
+ copy_whole_file(fm->filename, outfn);
+ cleanup_filemaps(filemaps, fmindex + 1);
+ return;
+ }
+
+ /* Write all blocks to the output file */
+
+ if (fstat(fileno(fm->fp), &statbuf) != 0)
+ {
+ pg_log_error("could not stat file \"%s\": %m", fm->filename);
+ pg_free(filemaps);
+ exit(1);
+ }
+
+ Assert((statbuf.st_size % BLCKSZ) == 0);
+
+ nblocks = statbuf.st_size / BLCKSZ;
+ if ((nblocks - 1) > lastblkno)
+ lastblkno = nblocks - 1;
+
+ outfp = fopen(outfn, "wb");
+ if (!outfp)
+ {
+ pg_log_error("could not create file \"%s\": %m", outfn);
+ cleanup_filemaps(filemaps, fmindex + 1);
+ exit(1);
+ }
+
+ for (i = 0; i <= lastblkno; i++)
+ {
+ char blkdata[BLCKSZ];
+ FILE *infp;
+ int offset;
+
+ /*
+ * Read block by block from respective file. If outblock has NULL
+ * file pointer, then fetch that block from the base file.
+ */
+ if (outblocks[i].fp != NULL)
+ {
+ infp = outblocks[i].fp;
+ offset = outblocks[i].offset;
+ }
+ else
+ {
+ infp = fm->fp;
+ offset = i * BLCKSZ;
+ }
+
+ if (fseek(infp, offset, SEEK_SET) == -1)
+ {
+ pg_log_error("could not fseek in file: %m");
+ cleanup_filemaps(filemaps, fmindex + 1);
+ exit(1);
+ }
+
+ if (fread(blkdata, 1, BLCKSZ, infp) != BLCKSZ)
+ {
+ pg_log_error("could not read from file \"%s\": %m", outfn);
+ cleanup_filemaps(filemaps, fmindex + 1);
+ exit(1);
+ }
+
+ /* Finally write one block to the output file */
+ if (fwrite(blkdata, 1, BLCKSZ, outfp) != BLCKSZ)
+ {
+ pg_log_error("could not write to file \"%s\": %m", outfn);
+ cleanup_filemaps(filemaps, fmindex + 1);
+ exit(1);
+ }
+ }
+
+ fclose(outfp);
+ cleanup_filemaps(filemaps, fmindex + 1);
+
+ return;
+}
+
+static void
+cleanup_filemaps(FileMap *filemaps, int nfilemaps)
+{
+ int i;
+
+ for (i = 0; i < nfilemaps; i++)
+ fclose(filemaps[i].fp);
+
+ pg_free(filemaps);
+}
--
1.8.3.1
From 740c153a712939d6c65f7d31592b75512544f383 Mon Sep 17 00:00:00 2001
From: Jeevan Chalke <jeevan.cha...@enterprisedb.com>
Date: Fri, 16 Aug 2019 14:10:33 +0530
Subject: [PATCH 3/4] Add support for the incremental backup.
If file is modified 90% or more, we send a whole file else we send
only those blocks which are modified. The file is named .partial and
has following header details:
- magic number, currently 0 (4 bytes)
- checksum, currently 0 (4 bytes)
- number of blocks in this .partial file (4 bytes)
- all modified block numbers (4 bytes each)
- modified blocks
---
doc/src/sgml/protocol.sgml | 50 ++++++-
doc/src/sgml/ref/pg_basebackup.sgml | 21 +++
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xlogfuncs.c | 6 +-
src/backend/replication/basebackup.c | 244 +++++++++++++++++++++++++++++++--
src/backend/storage/file/fd.c | 29 ++++
src/include/access/xlog.h | 3 +-
src/include/replication/basebackup.h | 13 ++
src/include/storage/fd.h | 1 +
9 files changed, 354 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b20f169..fb07585 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2466,7 +2466,7 @@ The commands accepted in replication mode are:
</varlistentry>
<varlistentry>
- <term><literal>BASE_BACKUP</literal> [ <literal>LABEL</literal> <replaceable>'label'</replaceable> ] [ <literal>PROGRESS</literal> ] [ <literal>FAST</literal> ] [ <literal>WAL</literal> ] [ <literal>NOWAIT</literal> ] [ <literal>MAX_RATE</literal> <replaceable>rate</replaceable> ] [ <literal>TABLESPACE_MAP</literal> ] [ <literal>NOVERIFY_CHECKSUMS</literal> ]
+ <term><literal>BASE_BACKUP</literal> [ <literal>LABEL</literal> <replaceable>'label'</replaceable> ] [ <literal>PROGRESS</literal> ] [ <literal>FAST</literal> ] [ <literal>WAL</literal> ] [ <literal>NOWAIT</literal> ] [ <literal>MAX_RATE</literal> <replaceable>rate</replaceable> ] [ <literal>TABLESPACE_MAP</literal> ] [ <literal>NOVERIFY_CHECKSUMS</literal> ] [ <literal>LSN</literal> <replaceable>'lsn'</replaceable> ]
<indexterm><primary>BASE_BACKUP</primary></indexterm>
</term>
<listitem>
@@ -2576,6 +2576,22 @@ The commands accepted in replication mode are:
</para>
</listitem>
</varlistentry>
+
+ <varlistentry>
+ <term><literal>LSN</literal> <replaceable>'lsn'</replaceable></term>
+ <listitem>
+ <para>
+ Includes only those data blocks in backup which has LSN greater than
+ or equal to the given lsn. However, if 90% or more data blocks are
+ modified in the file, then sends the entire file. Otherwise, creates
+ a <filename>.partial</filename> file containing only the blocks which
+ are modified and sends that instead. The <filename>.partial</filename>
+ file has its own header followed by the actual data blocks. Note that
+ only relation files are considered here, all other files are sent as
+ is.
+ </para>
+ </listitem>
+ </varlistentry>
</variablelist>
</para>
<para>
@@ -2698,6 +2714,38 @@ The commands accepted in replication mode are:
Owner, group, and file mode are set if the underlying file system on
the server supports it.
</para>
+ <para>
+ An incremental backup's <filename>.partial</filename> file has the
+ following format:
+ <itemizedlist>
+ <listitem>
+ <para>
+ Starts with a 4-byte magic number
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Followed by a 4-byte CRC of the header (containing a magic number,
+ count of the number of blocks, and all block numbers)
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Then a 4-byte count of the number of blocks included in the file
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Then the block numbers, each as a 4-byte quantity
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Followed by the actual data blocks in order with the block numbers
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
</listitem>
</varlistentry>
</variablelist>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index fc9e222..00782e0 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -408,6 +408,19 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
+ <term><option>--lsn=<replaceable class="parameter">LSN</replaceable></option></term>
+ <listitem>
+ <para>
+ Takes an incremental backup, using LSN as a threshold. Only the blocks
+ which are modified after this given LSN will be backed up. The file
+ which has these partial blocks has .partial as an extension. Backup
+ taken in this manner has to be combined with the full backup with the
+ <command>pg_combinebackup</command> utility.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-n</option></term>
<term><option>--no-clean</option></term>
<listitem>
@@ -792,6 +805,14 @@ PostgreSQL documentation
<prompt>$</prompt> <userinput>pg_basebackup -D backup/data -T /opt/ts=$(pwd)/backup/ts</userinput>
</screen>
</para>
+
+ <para>
+ To create an incremental backup having LSN greater than or equal to
+ <literal>5/19000060</literal>:
+<screen>
+<prompt>$</prompt> <userinput>pg_basebackup -D incbackup --lsn='5/19000060'</userinput>
+</screen>
+ </para>
</refsect1>
<refsect1>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f553523..e427c0f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -10178,7 +10178,7 @@ XLogRecPtr
do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
StringInfo labelfile, List **tablespaces,
StringInfo tblspcmapfile, bool infotbssize,
- bool needtblspcmapfile)
+ bool needtblspcmapfile, XLogRecPtr ref_lsn)
{
bool exclusive = (labelfile == NULL);
bool backup_started_in_recovery = false;
@@ -10506,6 +10506,9 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
appendStringInfo(labelfile, "START TIME: %s\n", strfbuf);
appendStringInfo(labelfile, "LABEL: %s\n", backupidstr);
appendStringInfo(labelfile, "START TIMELINE: %u\n", starttli);
+ if (!XLogRecPtrIsInvalid(ref_lsn))
+ appendStringInfo(labelfile, "INCREMENTAL BACKUP REFERENCE WAL LOCATION: %X/%X\n",
+ (uint32) (ref_lsn >> 32), (uint32) ref_lsn);
/*
* Okay, write the file, or return its contents to caller.
diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c
index b35043b..ef8b283 100644
--- a/src/backend/access/transam/xlogfuncs.c
+++ b/src/backend/access/transam/xlogfuncs.c
@@ -89,7 +89,8 @@ pg_start_backup(PG_FUNCTION_ARGS)
if (exclusive)
{
startpoint = do_pg_start_backup(backupidstr, fast, NULL, NULL,
- NULL, NULL, false, true);
+ NULL, NULL, false, true,
+ InvalidXLogRecPtr);
}
else
{
@@ -105,7 +106,8 @@ pg_start_backup(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
startpoint = do_pg_start_backup(backupidstr, fast, NULL, label_file,
- NULL, tblspc_map_file, false, true);
+ NULL, tblspc_map_file, false, true,
+ InvalidXLogRecPtr);
before_shmem_exit(nonexclusive_base_backup_cleanup, (Datum) 0);
}
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 18e992c..a2c0756 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -82,6 +82,12 @@ static pgoff_t do_full_backup(const char *readfilename,
const char *tarfilename, FILE *fp,
struct stat *statbuf, int segmentno,
bool verify_checksum, int *checksum_failures);
+static pgoff_t do_incremental_backup(const char *readfilename,
+ const char *tarfilename, FILE *fp,
+ XLogRecPtr refptr,
+ struct stat *statbuf, int segmentno,
+ bool verify_checksum,
+ int *checksum_failures);
/* Was the backup currently in-progress initiated in recovery mode? */
static bool backup_started_in_recovery = false;
@@ -99,6 +105,11 @@ static char *statrelpath = NULL;
*/
#define THROTTLING_FREQUENCY 8
+/*
+ * When to send the whole file, % blocks modified (90%)
+ */
+#define WHOLE_FILE_THRESHOLD 0.9
+
/* The actual number of bytes, transfer of which may cause sleep. */
static uint64 throttling_sample;
@@ -114,6 +125,9 @@ static TimestampTz throttled_last;
/* The starting XLOG position of the base backup. */
static XLogRecPtr startptr;
+/* The reference XLOG position for the incremental backup. */
+static XLogRecPtr refptr;
+
/* Total number of checksum failures during base backup. */
static long long int total_checksum_failures;
@@ -254,7 +268,9 @@ perform_base_backup(basebackup_options *opt)
startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
labelfile, &tablespaces,
tblspc_map_file,
- opt->progress, opt->sendtblspcmapfile);
+ opt->progress, opt->sendtblspcmapfile,
+ opt->lsn);
+ refptr = opt->lsn;
/*
* Once do_pg_start_backup has been called, ensure that any failure causes
@@ -1392,6 +1408,7 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
int segmentno = 0;
char *segmentpath;
bool verify_checksum = false;
+ char *filename;
fp = AllocateFile(readfilename, "rb");
if (fp == NULL)
@@ -1403,17 +1420,15 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
errmsg("could not open file \"%s\": %m", readfilename)));
}
+ /*
+ * Get the filename (excluding path). As last_dir_separator() includes
+ * the last directory separator, we chop that off by incrementing the
+ * pointer.
+ */
+ filename = last_dir_separator(readfilename) + 1;
+
if (!noverify_checksums && DataChecksumsEnabled())
{
- char *filename;
-
- /*
- * Get the filename (excluding path). As last_dir_separator()
- * includes the last directory separator, we chop that off by
- * incrementing the pointer.
- */
- filename = last_dir_separator(readfilename) + 1;
-
if (is_checksummed_file(readfilename, filename))
{
verify_checksum = true;
@@ -1434,9 +1449,23 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
}
}
- /* Perform full backup */
- len = do_full_backup(readfilename, tarfilename, fp, statbuf, segmentno,
- verify_checksum, &checksum_failures);
+ /*
+ * If incremental backup, see whether the filename is a relation filename
+ * or not.
+ */
+ if (refptr && OidIsValid(dboid) && looks_like_rel_name(filename))
+ {
+ /* Perform incremental backup */
+ len = do_incremental_backup(readfilename, tarfilename, fp,
+ refptr, statbuf, segmentno,
+ verify_checksum, &checksum_failures);
+ }
+ else
+ {
+ /* Perform full backup */
+ len = do_full_backup(readfilename, tarfilename, fp, statbuf, segmentno,
+ verify_checksum, &checksum_failures);
+ }
/* If the file was truncated while we were sending it, pad it with zeros */
if (len < statbuf->st_size)
@@ -1775,3 +1804,192 @@ do_full_backup(const char *readfilename, const char *tarfilename, FILE *fp,
return len;
}
+
+/*
+ * do_incremental_backup
+ *
+ * Perform incremental backup.
+ */
+static pgoff_t
+do_incremental_backup(const char *readfilename, const char *tarfilename,
+ FILE *fp, XLogRecPtr refptr, struct stat *statbuf,
+ int segmentno, bool verify_checksum,
+ int *checksum_failures)
+{
+ char *buf;
+ off_t cnt;
+ pgoff_t len = 0;
+ BlockNumber blkno = 0;
+ int i;
+ bool sendwholefile = false;
+
+ Assert(statbuf->st_size <= (RELSEG_SIZE * BLCKSZ));
+
+ buf = (char *) malloc(statbuf->st_size);
+ if (buf == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory")));
+
+ if ((cnt = fread(buf, 1, statbuf->st_size, fp)) > 0)
+ {
+ Bitmapset *mod_blocks = NULL;
+ int nmodblocks = 0;
+
+ if (cnt % BLCKSZ != 0)
+ {
+ if (verify_checksum)
+ {
+ ereport(WARNING,
+ (errmsg("cannot verify checksum in file \"%s\", block "
+ "%d: read buffer size %d and page size %d "
+ "differ",
+ readfilename, blkno, (int) cnt, BLCKSZ)));
+ verify_checksum = false;
+ }
+
+ ereport(WARNING,
+ (errmsg("file size (%d) not in multiple of page size (%d), sending whole file",
+ (int) cnt, BLCKSZ)));
+
+ /* File size is not in multiple of BLCKSZ, send as is. */
+ sendwholefile = true;
+ }
+
+ /*
+ * Check each page LSN and see if it is modified after the given LSN or
+ * not. Create a bitmap of all such modified blocks and then decide
+ * whether we want to send a whole file or a partial file. Skip this
+ * check if we decided to send whole file already.
+ */
+ if (!sendwholefile)
+ {
+ XLogRecPtr pglsn;
+ int nblocks = (cnt / BLCKSZ);
+
+ for (i = 0; i < nblocks; i++)
+ {
+ char *page = buf + BLCKSZ * i;
+
+ pglsn = PageGetLSN(page);
+
+ if (pglsn >= refptr)
+ {
+ /*
+ * Verify checksum, if requested, for the modified blocks.
+ */
+ if (verify_checksum)
+ verify_page_checksum(readfilename, fp, buf, cnt, i,
+ blkno, segmentno,
+ checksum_failures);
+
+ mod_blocks = bms_add_member(mod_blocks, i);
+ }
+
+ blkno++;
+ }
+
+ nmodblocks = bms_num_members(mod_blocks);
+
+ /*
+ * We need to send whole file if the modified block count is equal
+ * to or greater than the WHOLE_FILE_THRESHOLD. Check that.
+ */
+ if (i > 0 && (nmodblocks / (double) i) >= WHOLE_FILE_THRESHOLD)
+ sendwholefile = true;
+ }
+
+ /*
+ * If sendwholefile is true then we need to send the whole file as is.
+ * Otherwise send a partial file.
+ */
+ if (sendwholefile)
+ {
+ _tarWriteHeader(tarfilename, NULL, statbuf, false);
+
+ /* Send the chunk as a CopyData message */
+ if (pq_putmessage('d', buf, cnt))
+ ereport(ERROR,
+ (errmsg("base backup could not send data, aborting backup")));
+
+ len = cnt;
+ throttle(cnt);
+ }
+ else
+ {
+ int part_size = 0;
+ int part_header_size;
+ int blknum;
+ int blknocnt;
+ partial_file_header *pfh;
+ char *partialtarfilename = NULL;
+
+ /* Create a partial file */
+
+ /* Calculate partial file size. */
+ part_header_size = offsetof(partial_file_header, blocknumbers) +
+ (sizeof(uint32) * nmodblocks);
+ part_size = part_header_size + (BLCKSZ * nmodblocks);
+
+ /* Add .partial to filename */
+ partialtarfilename = (char *) palloc(strlen(tarfilename) + 9);
+ snprintf(partialtarfilename, strlen(tarfilename) + 9, "%s.partial", tarfilename);
+
+ statbuf->st_size = part_size;
+ _tarWriteHeader(partialtarfilename, NULL, statbuf, false);
+ pfree(partialtarfilename);
+
+ pfh = (partial_file_header *) palloc(part_header_size);
+ pfh->magic = INCREMENTAL_BACKUP_MAGIC;
+ pfh->nblocks = nmodblocks;
+
+ blknum = -1;
+ blknocnt = 0;
+ while ((blknum = bms_next_member(mod_blocks, blknum)) >= 0)
+ {
+ pfh->blocknumbers[blknocnt] = blknum;
+ /* Calculate CRC for each block to be transferred. */
+ blknocnt++;
+ }
+
+ Assert(blknocnt == nmodblocks);
+
+ /* Now calculate CRC for the header */
+ INIT_CRC32C(pfh->checksum);
+ COMP_CRC32C(pfh->checksum, pfh, part_header_size);
+
+ /* Send header */
+ if (pq_putmessage('d', (char *) pfh, part_header_size))
+ ereport(ERROR,
+ (errmsg("base backup could not send data, aborting backup")));
+ throttle(part_header_size);
+
+ /* Send data blocks */
+ for (blknocnt = 0; blknocnt < nmodblocks; blknocnt++)
+ {
+ int offset = BLCKSZ * pfh->blocknumbers[blknocnt];
+
+ if (pq_putmessage('d', buf + offset, BLCKSZ))
+ ereport(ERROR,
+ (errmsg("base backup could not send data, aborting backup")));
+ throttle(BLCKSZ);
+ }
+
+ Assert(blknocnt == nmodblocks);
+
+ len = part_size;
+ pfree(pfh);
+ }
+ }
+ else
+ {
+ /* Send empty file as is */
+ _tarWriteHeader(tarfilename, NULL, statbuf, false);
+ len = cnt;
+ }
+
+ /* free buffer allocated */
+ free(buf);
+
+ return len;
+}
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index a76112d..2990c52 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -3111,6 +3111,35 @@ looks_like_temp_rel_name(const char *name)
return true;
}
+/* <digits>, or <digits>.<digits> */
+bool
+looks_like_rel_name(const char *name)
+{
+ int pos;
+
+ /* Look for a non-empty string of digits (that isn't too long). */
+ for (pos = 0; isdigit((unsigned char) name[pos]); ++pos)
+ ;
+ if (pos == 0 || pos > OIDCHARS)
+ return false;
+
+ if (name[pos] == '.')
+ {
+ int segchar;
+
+ for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar)
+ ;
+ if (segchar <= 1)
+ return false;
+ pos += segchar;
+ }
+
+ /* Now we should be at the end. */
+ if (name[pos] != '\0')
+ return false;
+ return true;
+}
+
/*
* Issue fsync recursively on PGDATA and all its contents.
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252..155385d 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -347,7 +347,8 @@ typedef enum SessionBackupState
extern XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast,
TimeLineID *starttli_p, StringInfo labelfile,
List **tablespaces, StringInfo tblspcmapfile, bool infotbssize,
- bool needtblspcmapfile);
+ bool needtblspcmapfile,
+ XLogRecPtr ref_lsn);
extern XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive,
TimeLineID *stoptli_p);
extern void do_pg_abort_backup(void);
diff --git a/src/include/replication/basebackup.h b/src/include/replication/basebackup.h
index 503a5b9..1b35b08 100644
--- a/src/include/replication/basebackup.h
+++ b/src/include/replication/basebackup.h
@@ -20,6 +20,9 @@
#define MAX_RATE_LOWER 32
#define MAX_RATE_UPPER 1048576
+/* magic number in incremental backup's .partial file */
+#define INCREMENTAL_BACKUP_MAGIC 0x494E4352
+
typedef struct
{
@@ -29,6 +32,16 @@ typedef struct
int64 size;
} tablespaceinfo;
+/* Definition of the partial file header */
+typedef struct
+{
+ uint32 magic;
+ pg_crc32c checksum;
+ uint32 nblocks;
+ uint32 blocknumbers[FLEXIBLE_ARRAY_MEMBER];
+} partial_file_header;
+
+
extern void SendBaseBackup(BaseBackupCmd *cmd);
extern int64 sendTablespace(char *path, bool sizeonly);
diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index d2a8c52..d25a390 100644
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -136,6 +136,7 @@ extern void AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid,
SubTransactionId parentSubid);
extern void RemovePgTempFiles(void);
extern bool looks_like_temp_rel_name(const char *name);
+extern bool looks_like_rel_name(const char *name);
extern int pg_fsync(int fd);
extern int pg_fsync_no_writethrough(int fd);
--
1.8.3.1
From 2f55219552f30c2cc5a97b15f855fa402d99a1fd Mon Sep 17 00:00:00 2001
From: Jeevan Chalke <jeevan.cha...@enterprisedb.com>
Date: Fri, 16 Aug 2019 14:10:16 +0530
Subject: [PATCH 2/4] Refactor code in basebackup.c
- Refactor full backup code to the separate function.
- Refactor checksum varifying logic to the separate function.
---
src/backend/replication/basebackup.c | 308 ++++++++++++++++++++---------------
1 file changed, 176 insertions(+), 132 deletions(-)
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 74c954b..18e992c 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -75,6 +75,13 @@ static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
static int compareWalFileNames(const ListCell *a, const ListCell *b);
static void throttle(size_t increment);
static bool is_checksummed_file(const char *fullpath, const char *filename);
+static void verify_page_checksum(const char *readfilename, FILE *fp, char *buf,
+ off_t cnt, int blkindex, BlockNumber blkno, int segmentno,
+ int *checksum_failures);
+static pgoff_t do_full_backup(const char *readfilename,
+ const char *tarfilename, FILE *fp,
+ struct stat *statbuf, int segmentno,
+ bool verify_checksum, int *checksum_failures);
/* Was the backup currently in-progress initiated in recovery mode? */
static bool backup_started_in_recovery = false;
@@ -1377,17 +1384,11 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
bool missing_ok, Oid dboid)
{
FILE *fp;
- BlockNumber blkno = 0;
- bool block_retry = false;
char buf[TAR_SEND_SIZE];
- uint16 checksum;
int checksum_failures = 0;
off_t cnt;
- int i;
pgoff_t len = 0;
- char *page;
size_t pad;
- PageHeader phdr;
int segmentno = 0;
char *segmentpath;
bool verify_checksum = false;
@@ -1402,8 +1403,6 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
errmsg("could not open file \"%s\": %m", readfilename)));
}
- _tarWriteHeader(tarfilename, NULL, statbuf, false);
-
if (!noverify_checksums && DataChecksumsEnabled())
{
char *filename;
@@ -1435,130 +1434,9 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
}
}
- while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
- {
- /*
- * The checksums are verified at block level, so we iterate over the
- * buffer in chunks of BLCKSZ, after making sure that
- * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
- * BLCKSZ bytes.
- */
- Assert(TAR_SEND_SIZE % BLCKSZ == 0);
-
- if (verify_checksum && (cnt % BLCKSZ != 0))
- {
- ereport(WARNING,
- (errmsg("cannot verify checksum in file \"%s\", block "
- "%d: read buffer size %d and page size %d "
- "differ",
- readfilename, blkno, (int) cnt, BLCKSZ)));
- verify_checksum = false;
- }
-
- if (verify_checksum)
- {
- for (i = 0; i < cnt / BLCKSZ; i++)
- {
- page = buf + BLCKSZ * i;
-
- /*
- * Only check pages which have not been modified since the
- * start of the base backup. Otherwise, they might have been
- * written only halfway and the checksum would not be valid.
- * However, replaying WAL would reinstate the correct page in
- * this case. We also skip completely new pages, since they
- * don't have a checksum yet.
- */
- if (!PageIsNew(page) && PageGetLSN(page) < startptr)
- {
- checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
- phdr = (PageHeader) page;
- if (phdr->pd_checksum != checksum)
- {
- /*
- * Retry the block on the first failure. It's
- * possible that we read the first 4K page of the
- * block just before postgres updated the entire block
- * so it ends up looking torn to us. We only need to
- * retry once because the LSN should be updated to
- * something we can ignore on the next pass. If the
- * error happens again then it is a true validation
- * failure.
- */
- if (block_retry == false)
- {
- /* Reread the failed block */
- if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
- {
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not fseek in file \"%s\": %m",
- readfilename)));
- }
-
- if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
- {
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not reread block %d of file \"%s\": %m",
- blkno, readfilename)));
- }
-
- if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
- {
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not fseek in file \"%s\": %m",
- readfilename)));
- }
-
- /* Set flag so we know a retry was attempted */
- block_retry = true;
-
- /* Reset loop to validate the block again */
- i--;
- continue;
- }
-
- checksum_failures++;
-
- if (checksum_failures <= 5)
- ereport(WARNING,
- (errmsg("checksum verification failed in "
- "file \"%s\", block %d: calculated "
- "%X but expected %X",
- readfilename, blkno, checksum,
- phdr->pd_checksum)));
- if (checksum_failures == 5)
- ereport(WARNING,
- (errmsg("further checksum verification "
- "failures in file \"%s\" will not "
- "be reported", readfilename)));
- }
- }
- block_retry = false;
- blkno++;
- }
- }
-
- /* Send the chunk as a CopyData message */
- if (pq_putmessage('d', buf, cnt))
- ereport(ERROR,
- (errmsg("base backup could not send data, aborting backup")));
-
- len += cnt;
- throttle(cnt);
-
- if (len >= statbuf->st_size)
- {
- /*
- * Reached end of file. The file could be longer, if it was
- * extended while we were sending it, but for a base backup we can
- * ignore such extended data. It will be restored from WAL.
- */
- break;
- }
- }
+ /* Perform full backup */
+ len = do_full_backup(readfilename, tarfilename, fp, statbuf, segmentno,
+ verify_checksum, &checksum_failures);
/* If the file was truncated while we were sending it, pad it with zeros */
if (len < statbuf->st_size)
@@ -1731,3 +1609,169 @@ throttle(size_t increment)
*/
throttled_last = GetCurrentTimestamp();
}
+
+/*
+ * verify_page_checksum
+ *
+ * Verifies checksum for one page.
+ */
+static void
+verify_page_checksum(const char *readfilename, FILE *fp, char *buf,
+ off_t cnt, int blkindex, BlockNumber blkno, int segmentno,
+ int *checksum_failures)
+{
+ char *page;
+ uint16 checksum;
+ bool block_retry = false;
+
+ while (1)
+ {
+ page = buf + BLCKSZ * blkindex;
+
+ /*
+ * Only check pages which have not been modified since the start of the
+ * base backup. Otherwise, they might have been written only halfway
+ * and the checksum would not be valid. However, replaying WAL would
+ * reinstate the correct page in this case. We also skip completely
+ * new pages, since they don't have a checksum yet.
+ */
+ if (!PageIsNew(page) && PageGetLSN(page) < startptr)
+ {
+ PageHeader phdr;
+
+ checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
+ phdr = (PageHeader) page;
+ if (phdr->pd_checksum != checksum)
+ {
+ /*
+ * Retry the block on the first failure. It's possible that we
+ * read the first 4K page of the block just before postgres
+ * updated the entire block so it ends up looking torn to us.
+ * We only need to retry once because the LSN should be updated
+ * to something we can ignore on the next pass. If the error
+ * happens again then it is a true validation failure.
+ */
+ if (block_retry == false)
+ {
+ /* Reread the failed block */
+ if (fseek(fp, -(cnt - BLCKSZ * blkindex), SEEK_CUR) == -1)
+ {
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not fseek in file \"%s\": %m",
+ readfilename)));
+ }
+
+ if (fread(buf + BLCKSZ * blkindex, 1, BLCKSZ, fp) != BLCKSZ)
+ {
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not reread block %d of file \"%s\": %m",
+ blkno, readfilename)));
+ }
+
+ if (fseek(fp, cnt - BLCKSZ * blkindex - BLCKSZ, SEEK_CUR) == -1)
+ {
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not fseek in file \"%s\": %m",
+ readfilename)));
+ }
+
+ /* Set flag so we know a retry was attempted */
+ block_retry = true;
+
+ /* Re-validate the block again */
+ continue;
+ }
+
+ (*checksum_failures)++;
+
+ if (*checksum_failures <= 5)
+ ereport(WARNING,
+ (errmsg("checksum verification failed in "
+ "file \"%s\", block %d: calculated "
+ "%X but expected %X",
+ readfilename, blkno, checksum,
+ phdr->pd_checksum)));
+ if (*checksum_failures == 5)
+ ereport(WARNING,
+ (errmsg("further checksum verification "
+ "failures in file \"%s\" will not "
+ "be reported", readfilename)));
+ }
+ }
+
+ break;
+ }
+}
+
+/*
+ * do_full_backup
+ *
+ * Perform full backup.
+ */
+static pgoff_t
+do_full_backup(const char *readfilename, const char *tarfilename, FILE *fp,
+ struct stat *statbuf, int segmentno, bool verify_checksum,
+ int *checksum_failures)
+{
+ char buf[TAR_SEND_SIZE];
+ off_t cnt;
+ pgoff_t len = 0;
+ BlockNumber blkno = 0;
+ int i;
+
+ _tarWriteHeader(tarfilename, NULL, statbuf, false);
+
+ while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
+ {
+ /*
+ * The checksums are verified at block level, so we iterate over the
+ * buffer in chunks of BLCKSZ, after making sure that
+ * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
+ * BLCKSZ bytes.
+ */
+ Assert(TAR_SEND_SIZE % BLCKSZ == 0);
+
+ if (verify_checksum && (cnt % BLCKSZ != 0))
+ {
+ ereport(WARNING,
+ (errmsg("cannot verify checksum in file \"%s\", block "
+ "%d: read buffer size %d and page size %d "
+ "differ",
+ readfilename, blkno, (int) cnt, BLCKSZ)));
+ verify_checksum = false;
+ }
+
+ if (verify_checksum)
+ {
+ for (i = 0; i < cnt / BLCKSZ; i++)
+ {
+ verify_page_checksum(readfilename, fp, buf, cnt, i, blkno,
+ segmentno, checksum_failures);
+ blkno++;
+ }
+ }
+
+ /* Send the chunk as a CopyData message */
+ if (pq_putmessage('d', buf, cnt))
+ ereport(ERROR,
+ (errmsg("base backup could not send data, aborting backup")));
+
+ len += cnt;
+ throttle(cnt);
+
+ if (len >= statbuf->st_size)
+ {
+ /*
+ * Reached end of file. The file could be longer, if it was
+ * extended while we were sending it, but for a base backup we can
+ * ignore such extended data. It will be restored from WAL.
+ */
+ break;
+ }
+ }
+
+ return len;
+}
--
1.8.3.1
