I've been working on your patch. Attached is a version I'd be happy to
commit. Please check that it's okay with you.
I rewrote the option argument parsing logic a little bit to be more
clear and provide more specific error messages.
I reinstated the requirement that both old and new directory are
absolute. After consideration, I think this makes sense because all
tablespace directories are always required to be absolute in other
contexts. (Note: Checking for absolute path by testing the first
character for '/' is not portable.)
I also removed the partial matching. This would have let -T /data1=...
also match /data11, which is clearly confusing. This logic would need
some intelligence about slashes, similar to fnmatch(). This could
perhaps be added later.
Finally, I wrote some test cases for this new functionality. See the
attached patch, which can be applied on top of
<https://commitfest.postgresql.org/action/patch_view?id=1394>.
>From cc189020d04ff2311c92108620e4fc74f80c01c9 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pete...@gmx.net>
Date: Sat, 15 Feb 2014 08:42:20 -0500
Subject: [PATCH] pg_basebackup: Add support for relocating tablespaces
Tablespaces can be relocated in plain backup mode by specifying one or
more -T olddir=newdir options.
From: Steeve Lennmark <stee...@handeldsbanken.se>
Reviewed-by: Peter Eisentraut <pete...@gmx.net>
---
doc/src/sgml/ref/pg_basebackup.sgml | 46 +++++++++-
src/bin/pg_basebackup/pg_basebackup.c | 166 +++++++++++++++++++++++++++++++++-
2 files changed, 204 insertions(+), 8 deletions(-)
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index c379df5..ea22331 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -203,6 +203,33 @@ <title>Options</title>
</varlistentry>
<varlistentry>
+ <term><option>-T <replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>
+ <term><option>--tablespace-mapping=<replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>
+ <listitem>
+ <para>
+ Relocate the tablespace in directory <replaceable>olddir</replaceable>
+ to <replaceable>newdir</replaceable> during the backup. To be
+ effective, <replaceable>olddir</replaceable> must exactly match the
+ path specification of the tablespace as it is currently defined. (But
+ it is not an error if there is no tablespace
+ in <replaceable>olddir</replaceable> contained in the backup.)
+ Both <replaceable>olddir</replaceable>
+ and <replaceable>newdir</replaceable> must be absolute paths. If a
+ path happens to contain a <literal>=</literal> sign, escape it with a
+ backslash. This option can be specified multiple times for multiple
+ tablespaces. See examples below.
+ </para>
+
+ <para>
+ If a tablespace is relocated in this way, the symbolic links inside
+ the main data directory are updated to point to the new location. So
+ the new data directory is ready to be used for a new server instance
+ with all tablespaces in the updated locations.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>--xlogdir=<replaceable class="parameter">xlogdir</replaceable></option></term>
<listitem>
<para>
@@ -528,9 +555,13 @@ <title>Notes</title>
</para>
<para>
- The way <productname>PostgreSQL</productname> manages tablespaces, the path
- for all additional tablespaces must be identical whenever a backup is
- restored. The main data directory, however, is relocatable to any location.
+ Tablespaces will in plain format by default be backed up to the same path
+ they have on the server, unless the
+ option <replaceable>--tablespace-mapping</replaceable> is used. Without
+ this option, running a plain format base backup on the same host as the
+ server will not work if tablespaces are in use, because the backup would
+ have to be written to the same directory locations as the original
+ tablespaces.
</para>
<para>
@@ -570,6 +601,15 @@ <title>Examples</title>
(This command will fail if there are multiple tablespaces in the
database.)
</para>
+
+ <para>
+ To create a backup of a local database where the tablespace in
+ <filename>/opt/ts</filename> is relocated
+ to <filename>./backup/ts</filename>:
+<screen>
+<prompt>$</prompt> <userinput>pg_basebackup -D backup/data -T /opt/ts=$(pwd)/backup/ts</userinput>
+</screen>
+ </para>
</refsect1>
<refsect1>
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index b5682d6..b27678f 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -35,8 +35,24 @@
#include "streamutil.h"
+#define atooid(x) ((Oid) strtoul((x), NULL, 10))
+
+typedef struct TablespaceListCell
+{
+ struct TablespaceListCell *next;
+ char old_dir[MAXPGPATH];
+ char new_dir[MAXPGPATH];
+} TablespaceListCell;
+
+typedef struct TablespaceList
+{
+ TablespaceListCell *head;
+ TablespaceListCell *tail;
+} TablespaceList;
+
/* Global options */
static char *basedir = NULL;
+static TablespaceList tablespace_dirs = {NULL, NULL};
static char *xlog_dir = "";
static char format = 'p'; /* p(lain)/t(ar) */
static char *label = "pg_basebackup base backup";
@@ -90,6 +106,10 @@
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
bool segment_finished);
+static const char *get_tablespace_mapping(const char *dir);
+static void update_tablespace_symlink(Oid oid, const char *old_dir);
+static void tablespace_list_append(const char *arg);
+
static void disconnect_and_exit(int code)
{
@@ -110,6 +130,77 @@ static void disconnect_and_exit(int code)
}
+/*
+ * Split argument into old_dir and new_dir and append to tablespace mapping
+ * list.
+ */
+static void
+tablespace_list_append(const char *arg)
+{
+ TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell));
+ char *dst;
+ char *dst_ptr;
+ const char *arg_ptr;
+
+ dst_ptr = dst = cell->old_dir;
+ for (arg_ptr = arg; *arg_ptr; arg_ptr++)
+ {
+ if (dst_ptr - dst >= MAXPGPATH)
+ {
+ fprintf(stderr, _("%s: directory name too long\n"), progname);
+ exit(1);
+ }
+
+ if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
+ ; /* skip backslash escaping = */
+ else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
+ {
+ if (*cell->new_dir)
+ {
+ fprintf(stderr, _("%s: multiple \"=\" signs in tablespace mapping\n"), progname);
+ exit(1);
+ }
+ else
+ dst = dst_ptr = cell->new_dir;
+ }
+ else
+ *dst_ptr++ = *arg_ptr;
+ }
+
+ if (!*cell->old_dir || !*cell->new_dir)
+ {
+ fprintf(stderr,
+ _("%s: invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"\n"),
+ progname, arg);
+ exit(1);
+ }
+
+ /* This check isn't absolutely necessary. But all tablespaces are created
+ * with absolute directories, so specifying a non-absolute path here would
+ * just never match, possibly confusing users. It's also good to be
+ * consistent with the new_dir check. */
+ if (!is_absolute_path(cell->old_dir))
+ {
+ fprintf(stderr, _("%s: old directory not absolute in tablespace mapping: %s\n"),
+ progname, cell->old_dir);
+ exit(1);
+ }
+
+ if (!is_absolute_path(cell->new_dir))
+ {
+ fprintf(stderr, _("%s: new directory not absolute in tablespace mapping: %s\n"),
+ progname, cell->new_dir);
+ exit(1);
+ }
+
+ if (tablespace_dirs.tail)
+ tablespace_dirs.tail->next = cell;
+ else
+ tablespace_dirs.head = cell;
+ tablespace_dirs.tail = cell;
+}
+
+
#ifdef HAVE_LIBZ
static const char *
get_gz_error(gzFile gzf)
@@ -137,6 +228,8 @@ static void disconnect_and_exit(int code)
printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
printf(_(" -R, --write-recovery-conf\n"
" write recovery.conf after backup\n"));
+ printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
+ " relocate tablespace in OLDDIR to NEWDIR\n"));
printf(_(" -x, --xlog include required WAL files in backup (fetch mode)\n"));
printf(_(" -X, --xlog-method=fetch|stream\n"
" include required WAL files with specified method\n"));
@@ -899,6 +992,52 @@ static void disconnect_and_exit(int code)
PQfreemem(copybuf);
}
+
+/*
+ * Retrieve tablespace path, either relocated or original depending on whether
+ * -T was passed or not.
+ */
+static const char *
+get_tablespace_mapping(const char *dir)
+{
+ TablespaceListCell *cell;
+
+ for (cell = tablespace_dirs.head; cell; cell = cell->next)
+ if (strcmp(dir, cell->old_dir) == 0)
+ return cell->new_dir;
+
+ return dir;
+}
+
+
+/*
+ * Update symlinks to reflect relocated tablespace.
+ */
+static void
+update_tablespace_symlink(Oid oid, const char *old_dir)
+{
+ const char *new_dir = get_tablespace_mapping(old_dir);
+
+ if (strcmp(old_dir, new_dir) != 0)
+ {
+ char *linkloc = psprintf("%s/pg_tblspc/%d", basedir, oid);
+
+ if (unlink(linkloc) != 0 && errno != ENOENT)
+ {
+ fprintf(stderr, _("%s: could not remove symbolic link \"%s\": %s"),
+ progname, linkloc, strerror(errno));
+ disconnect_and_exit(1);
+ }
+ if (symlink(new_dir, linkloc) != 0)
+ {
+ fprintf(stderr, _("%s: could not create symbolic link \"%s\": %s"),
+ progname, linkloc, strerror(errno));
+ disconnect_and_exit(1);
+ }
+ }
+}
+
+
/*
* Receive a tar format stream from the connection to the server, and unpack
* the contents of it into a directory. Only files, directories and
@@ -906,8 +1045,7 @@ static void disconnect_and_exit(int code)
*
* If the data is for the main data directory, it will be restored in the
* specified directory. If it's for another tablespace, it will be restored
- * in the original directory, since relocation of tablespaces is not
- * supported.
+ * in the original or mapped directory.
*/
static void
ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
@@ -923,7 +1061,7 @@ static void disconnect_and_exit(int code)
if (basetablespace)
strcpy(current_path, basedir);
else
- strcpy(current_path, PQgetvalue(res, rownum, 1));
+ strcpy(current_path, get_tablespace_mapping(PQgetvalue(res, rownum, 1)));
/*
* Get the COPY data
@@ -1503,7 +1641,10 @@ static void disconnect_and_exit(int code)
* we do anything anyway.
*/
if (format == 'p' && !PQgetisnull(res, i, 1))
- verify_dir_is_empty_or_create(PQgetvalue(res, i, 1));
+ {
+ char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1));
+ verify_dir_is_empty_or_create(path);
+ }
}
/*
@@ -1545,6 +1686,17 @@ static void disconnect_and_exit(int code)
progress_report(PQntuples(res), NULL, true);
fprintf(stderr, "\n"); /* Need to move to next line */
}
+
+ if (format == 'p' && tablespace_dirs.head != NULL)
+ {
+ for (i = 0; i < PQntuples(res); i++)
+ {
+ Oid tblspc_oid = atooid(PQgetvalue(res, i, 0));
+ if (tblspc_oid)
+ update_tablespace_symlink(tblspc_oid, PQgetvalue(res, i, 1));
+ }
+ }
+
PQclear(res);
/*
@@ -1696,6 +1848,7 @@ static void disconnect_and_exit(int code)
{"format", required_argument, NULL, 'F'},
{"checkpoint", required_argument, NULL, 'c'},
{"write-recovery-conf", no_argument, NULL, 'R'},
+ {"tablespace-mapping", required_argument, NULL, 'T'},
{"xlog", no_argument, NULL, 'x'},
{"xlog-method", required_argument, NULL, 'X'},
{"gzip", no_argument, NULL, 'z'},
@@ -1735,7 +1888,7 @@ static void disconnect_and_exit(int code)
}
}
- while ((c = getopt_long(argc, argv, "D:F:RxX:l:zZ:d:c:h:p:U:s:wWvP",
+ while ((c = getopt_long(argc, argv, "D:F:RT:xX:l:zZ:d:c:h:p:U:s:wWvP",
long_options, &option_index)) != -1)
{
switch (c)
@@ -1759,6 +1912,9 @@ static void disconnect_and_exit(int code)
case 'R':
writerecoveryconf = true;
break;
+ case 'T':
+ tablespace_list_append(optarg);
+ break;
case 'x':
if (includewal)
{
--
1.8.5.5
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index b4c0471..cf7c4b6 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -2,7 +2,7 @@
use warnings;
use Cwd;
use TestLib;
-use Test::More tests => 19;
+use Test::More tests => 31;
program_help_ok('pg_basebackup');
program_version_ok('pg_basebackup');
@@ -48,6 +48,47 @@
note("tablespace tars are @tblspc_tars");
is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup1", '-Fp'],
+ 'plain format with tablespaces fails without tablespace mapping');
+
+command_ok(['pg_basebackup', '-D', "$tempdir/backup1", '-Fp',
+ "-T$tempdir/tblspc1=$tempdir/tbackup/tblspc1"],
+ 'plain format with tablespaces succeeds with tablespace mapping');
+ok(-d "$tempdir/tbackup/tblspc1", 'tablespace was relocated');
+opendir(my $dh, "$tempdir/pgdata/pg_tblspc") or die;
+ok((grep { -l "$tempdir/backup1/pg_tblspc/$_" and readlink "$tempdir/backup1/pg_tblspc/$_" eq "$tempdir/tbackup/tblspc1" } readdir($dh)),
+ "tablespace symlink was updated");
+closedir $dh;
+
+mkdir "$tempdir/tbl=spc2";
+psql 'postgres', "DROP TABLE test1;";
+psql 'postgres', "DROP TABLESPACE tblspc1;";
+psql 'postgres', "CREATE TABLESPACE tblspc2 LOCATION '$tempdir/tbl=spc2';";
+command_ok(['pg_basebackup', '-D', "$tempdir/backup3", '-Fp',
+ "-T$tempdir/tbl\\=spc2=$tempdir/tbackup/tbl\\=spc2"],
+ 'mapping tablespace with = sign in path');
+ok(-d "$tempdir/tbackup/tbl=spc2", 'tablespace with = sign was relocated');
+
+psql 'postgres', "DROP TABLESPACE tblspc2;";
+
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-T=/foo"],
+ '-T with empty old directory fails');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-T/foo="],
+ '-T with empty new directory fails');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-T/foo=/bar=/baz"],
+ '-T with multiple = fails');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-Tfoo=/bar"],
+ '-T with old directory not absolute fails');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-T/foo=bar"],
+ '-T with new directory not absolute fails');
+command_fails(['pg_basebackup', '-D', "$tempdir/backup_foo", '-Fp',
+ "-Tfoo"],
+ '-T with invalid format fails');
our $TODO = 'https://commitfest.postgresql.org/action/patch_view?id=1303';
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
