Including unlogged relations in base backups takes up space and is
wasteful since they are truncated during backup recovery.
The attached patches exclude unlogged relations from base backups except
for the init fork, which is required to recreate the main fork during
recovery.
* exclude-unlogged-v1-01.patch
Some refactoring of reinit.c was required to reduce code duplication but
the coverage report showed that most of the interesting parts of
reinit.c were not being tested. This patch adds coverage for reinit.c.
* exclude-unlogged-v1-02.patch
Refactor reinit.c to allow other modules to identify and work with
unlogged relation forks.
* exclude-unlogged-v1-03.patch
Exclude unlogged relation forks (except init) from pg_basebackup to save
space (and time).
I decided not to try and document unlogged exclusions in the continuous
backup documentation yet (they are noted in the protocol docs). I would
like to get some input on whether the community thinks this is a good
idea. It's a non-trivial procedure that would be easy to misunderstand
and does not affect the quality of the backup other than using less
space. Thoughts?
I'll add these patches to the next CF.
--
-David
da...@pgmasters.net
diff --git a/src/test/recovery/t/014_unlogged_reinit.pl
b/src/test/recovery/t/014_unlogged_reinit.pl
new file mode 100644
index 0000000000..35feba69a0
--- /dev/null
+++ b/src/test/recovery/t/014_unlogged_reinit.pl
@@ -0,0 +1,117 @@
+# Tests that unlogged tables are properly reinitialized after a crash.
+#
+# The behavior should be the same when restoring from a backup but that is not
+# tested here (yet).
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 16;
+
+# Initialize node without replication settings
+my $node = get_new_node('main');
+
+$node->init;
+$node->start;
+my $pgdata = $node->data_dir;
+
+# Create an unlogged table to test that forks other than init are not copied
+$node->safe_psql('postgres', 'CREATE UNLOGGED TABLE base_unlogged (id int)');
+
+my $baseUnloggedPath = $node->safe_psql('postgres',
+ q{select pg_relation_filepath('base_unlogged')});
+
+# Make sure main and init forks exist
+ok(-f "$pgdata/${baseUnloggedPath}_init", 'init fork in base');
+ok(-f "$pgdata/$baseUnloggedPath", 'main fork in base');
+
+# The following tests test symlinks. Windows doesn't have symlinks, so
+# skip on Windows.
+my $tablespaceDir = undef;
+my $ts1UnloggedPath = undef;
+
+SKIP:
+{
+ skip "symlinks not supported on Windows", 2 if ($windows_os);
+
+ # Create unlogged tables in a tablespace
+ $tablespaceDir = TestLib::tempdir . "/ts1";
+
+ mkdir($tablespaceDir)
+ or die "unable to mkdir \"$tablespaceDir\"";
+
+ $node->safe_psql('postgres',
+ "CREATE TABLESPACE ts1 LOCATION '$tablespaceDir'");
+ $node->safe_psql('postgres',
+ 'CREATE UNLOGGED TABLE ts1_unlogged (id int) TABLESPACE ts1');
+
+ $ts1UnloggedPath = $node->safe_psql('postgres',
+ q{select pg_relation_filepath('ts1_unlogged')});
+
+ # Make sure main and init forks exist
+ ok(-f "$pgdata/${ts1UnloggedPath}_init", 'init fork in tablespace');
+ ok(-f "$pgdata/$ts1UnloggedPath", 'main fork in tablespace');
+}
+
+# Crash the postmaster
+$node->stop('immediate');
+
+# Write forks to test that they are removed during recovery
+$node->command_ok(['touch', "$pgdata/${baseUnloggedPath}_vm"],
+ 'touch vm fork in base');
+$node->command_ok(['touch', "$pgdata/${baseUnloggedPath}_fsm"],
+ 'touch fsm fork in base');
+
+# Remove main fork to test that it is recopied from init
+unlink("$pgdata/${baseUnloggedPath}")
+ or die "unable to remove \"${baseUnloggedPath}\"";
+
+# The following tests test symlinks. Windows doesn't have symlinks, so
+# skip on Windows.
+SKIP:
+{
+ skip "symlinks not supported on Windows", 2 if ($windows_os);
+
+ # Write forks to test that they are removed by recovery
+ $node->command_ok(['touch', "$pgdata/${ts1UnloggedPath}_vm"],
+ 'touch vm fork in tablespace');
+ $node->command_ok(['touch', "$pgdata/${ts1UnloggedPath}_fsm"],
+ 'touch fsm fork in tablespace');
+
+ # Remove main fork to test that it is recopied from init
+ unlink("$pgdata/${ts1UnloggedPath}")
+ or die "unable to remove \"${ts1UnloggedPath}\"";
+}
+
+# Start the postmaster
+$node->start;
+
+# Check unlogged table in base
+ok(-f "$pgdata/${baseUnloggedPath}_init", 'init fork in base');
+ok(-f "$pgdata/$baseUnloggedPath", 'main fork in base');
+ok(!-f "$pgdata/${baseUnloggedPath}_vm", 'vm fork not in base');
+ok(!-f "$pgdata/${baseUnloggedPath}_fsm", 'fsm fork not in base');
+
+# Drop unlogged table
+$node->safe_psql('postgres', 'DROP TABLE base_unlogged');
+
+# The following tests test symlinks. Windows doesn't have symlinks, so
+# skip on Windows.
+SKIP:
+{
+ skip "symlinks not supported on Windows", 4 if ($windows_os);
+
+ # Check unlogged table in tablespace
+ ok(-f "$pgdata/${ts1UnloggedPath}_init", 'init fork in tablespace');
+ ok(-f "$pgdata/$ts1UnloggedPath", 'main fork in tablespace');
+ ok(!-f "$pgdata/${ts1UnloggedPath}_vm", 'vm fork not in tablespace');
+ ok(!-f "$pgdata/${ts1UnloggedPath}_fsm", 'fsm fork not in tablespace');
+
+ # Drop unlogged table
+ $node->safe_psql('postgres', 'DROP TABLE ts1_unlogged');
+
+ # Drop tablespace
+ $node->safe_psql('postgres', 'DROP TABLESPACE ts1');
+ rmdir($tablespaceDir)
+ or die "unable to rmdir \"$tablespaceDir\"";
+}
diff --git a/src/backend/storage/file/reinit.c
b/src/backend/storage/file/reinit.c
index 00678cb182..0c2568e3f5 100644
--- a/src/backend/storage/file/reinit.c
+++ b/src/backend/storage/file/reinit.c
@@ -21,7 +21,6 @@
#include "storage/copydir.h"
#include "storage/fd.h"
#include "storage/reinit.h"
-#include "utils/hsearch.h"
#include "utils/memutils.h"
static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
@@ -146,6 +145,99 @@ ResetUnloggedRelationsInTablespaceDir(const char
*tsdirname, int op)
}
/*
+ * Find all unlogged relations in the specified directory and return their
OIDs.
+ *
+ * It's possible that someone could create a ton of unlogged relations in the
+ * same database & tablespace, so we'd better use a hash table rather than an
+ * array or linked list to keep track of which files need to be reset.
+ * Otherwise, search operations would be O(n^2).
+ */
+HTAB *
+ResetUnloggedRelationsHash(const char *dbspacedirname)
+{
+ DIR *dbspace_dir;
+ struct dirent *de;
+ HTAB *hash = NULL;
+
+ /* Scan the directory. */
+ dbspace_dir = AllocateDir(dbspacedirname);
+ while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
+ {
+ ForkNumber forkNum;
+ int oidchars;
+ unlogged_relation_entry ent;
+
+ /* Skip anything that doesn't look like a relation data file. */
+ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
+
&forkNum))
+ continue;
+
+ /* Also skip it unless this is the init fork. */
+ if (forkNum != INIT_FORKNUM)
+ continue;
+
+ /* Create the hash table if it has not been created already. */
+ if (!hash)
+ {
+ HASHCTL ctl;
+
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(unlogged_relation_entry);
+ ctl.entrysize = sizeof(unlogged_relation_entry);
+ hash = hash_create("unlogged hash", 32, &ctl,
HASH_ELEM);
+ }
+
+ /*
+ * Put the OID portion of the name into the hash table, if it
+ * isn't already.
+ */
+ memset(ent.oid, 0, sizeof(ent.oid));
+ memcpy(ent.oid, de->d_name, oidchars);
+ hash_search(hash, &ent, HASH_ENTER, NULL);
+ }
+
+ /* Done with the first pass. */
+ FreeDir(dbspace_dir);
+
+ return hash;
+}
+
+/*
+ * Determine whether the specified file is an unlogged relation fork.
+ *
+ * If not an unlogged relation then return notUnlogged, otherwise return
+ * unloggedInit if an unlogged init fork and unloggedOther if any other
unlogged
+ * fork.
+ */
+UnloggedRelationFork
+ResetUnloggedRelationsMatch(HTAB *hash, const char *file)
+{
+ ForkNumber forkNum;
+ int oidchars;
+ bool found;
+ unlogged_relation_entry ent;
+
+ /* If it's not a relation then it's not unlogged. */
+ if (!parse_filename_for_nontemp_relation(file, &oidchars, &forkNum))
+ return notUnlogged;
+
+ /* An unlogged init fork. */
+ if (forkNum == INIT_FORKNUM)
+ return unloggedInit;
+
+ /* See whether the OID portion of the name shows up in the hash table.
*/
+ memset(ent.oid, 0, sizeof(ent.oid));
+ memcpy(ent.oid, file, oidchars);
+ hash_search(hash, &ent, HASH_FIND, &found);
+
+ /* If found this is another fork of an unlogged table (but not init). */
+ if (found)
+ return unloggedOther;
+
+ return notUnlogged;
+}
+
+/*
* Process one per-dbspace directory for ResetUnloggedRelations
*/
static void
@@ -166,58 +258,13 @@ ResetUnloggedRelationsInDbspaceDir(const char
*dbspacedirname, int op)
if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
{
HTAB *hash;
- HASHCTL ctl;
-
- /*
- * It's possible that someone could create a ton of unlogged
relations
- * in the same database & tablespace, so we'd better use a hash
table
- * rather than an array or linked list to keep track of which
files
- * need to be reset. Otherwise, this cleanup operation would be
- * O(n^2).
- */
- memset(&ctl, 0, sizeof(ctl));
- ctl.keysize = sizeof(unlogged_relation_entry);
- ctl.entrysize = sizeof(unlogged_relation_entry);
- hash = hash_create("unlogged hash", 32, &ctl, HASH_ELEM);
-
- /* Scan the directory. */
- dbspace_dir = AllocateDir(dbspacedirname);
- while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
- {
- ForkNumber forkNum;
- int oidchars;
- unlogged_relation_entry ent;
-
- /* Skip anything that doesn't look like a relation data
file. */
- if (!parse_filename_for_nontemp_relation(de->d_name,
&oidchars,
-
&forkNum))
- continue;
-
- /* Also skip it unless this is the init fork. */
- if (forkNum != INIT_FORKNUM)
- continue;
- /*
- * Put the OID portion of the name into the hash table,
if it
- * isn't already.
- */
- memset(ent.oid, 0, sizeof(ent.oid));
- memcpy(ent.oid, de->d_name, oidchars);
- hash_search(hash, &ent, HASH_ENTER, NULL);
- }
-
- /* Done with the first pass. */
- FreeDir(dbspace_dir);
+ /* Build a hash table of all unlogged relations. */
+ hash = ResetUnloggedRelationsHash(dbspacedirname);
- /*
- * If we didn't find any init forks, there's no point in
continuing;
- * we can bail out now.
- */
- if (hash_get_num_entries(hash) == 0)
- {
- hash_destroy(hash);
+ /* No need to continue if there are no unlogged tables. */
+ if (!hash)
return;
- }
/*
* Now, make a second pass and remove anything that matches.
@@ -225,30 +272,9 @@ ResetUnloggedRelationsInDbspaceDir(const char
*dbspacedirname, int op)
dbspace_dir = AllocateDir(dbspacedirname);
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
- ForkNumber forkNum;
- int oidchars;
- bool found;
- unlogged_relation_entry ent;
-
- /* Skip anything that doesn't look like a relation data
file. */
- if (!parse_filename_for_nontemp_relation(de->d_name,
&oidchars,
-
&forkNum))
- continue;
-
- /* We never remove the init fork. */
- if (forkNum == INIT_FORKNUM)
- continue;
-
- /*
- * See whether the OID portion of the name shows up in
the hash
- * table.
- */
- memset(ent.oid, 0, sizeof(ent.oid));
- memcpy(ent.oid, de->d_name, oidchars);
- hash_search(hash, &ent, HASH_FIND, &found);
-
- /* If so, nuke it! */
- if (found)
+ /* If this is an unlogged relation fork other than
init, nuke it! */
+ if (ResetUnloggedRelationsMatch(
+ hash, de->d_name) == unloggedOther)
{
snprintf(rm_path, sizeof(rm_path), "%s/%s",
dbspacedirname, de->d_name);
diff --git a/src/include/storage/reinit.h b/src/include/storage/reinit.h
index 90e494e933..f5781b5bf2 100644
--- a/src/include/storage/reinit.h
+++ b/src/include/storage/reinit.h
@@ -15,9 +15,24 @@
#ifndef REINIT_H
#define REINIT_H
+#include "utils/hsearch.h"
+
extern void ResetUnloggedRelations(int op);
#define UNLOGGED_RELATION_CLEANUP 0x0001
#define UNLOGGED_RELATION_INIT 0x0002
+/* Return values for ResetUnloggedRelationsMatch(). */
+typedef enum
+{
+ notUnlogged, /* Not a relation or not an unlogged
relation. */
+ unloggedInit, /* An unlogged relation init fork. */
+ unloggedOther /* An unlogged relation fork other than
init. */
+} UnloggedRelationFork;
+
+/* Utility functions for identifying unlogged table forks. */
+extern HTAB *ResetUnloggedRelationsHash(const char *dbspacedirname);
+extern UnloggedRelationFork ResetUnloggedRelationsMatch(
+ HTAB *unloggedHash, const char *fileName);
+
#endif /* REINIT_H */
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 8174e3defa..acac286bf4 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2549,6 +2549,12 @@ The commands accepted in walsender mode are:
</listitem>
<listitem>
<para>
+ Unlogged relations, except for the init fork which is required to
+ recreate the (empty) unlogged relation on recovery.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
<filename>pg_wal</filename>, including subdirectories. If the backup
is run
with WAL files included, a synthesized version of
<filename>pg_wal</filename> will be
included, but it will only contain the files necessary for the
diff --git a/src/backend/replication/basebackup.c
b/src/backend/replication/basebackup.c
index cd7d391b2f..59ab7e19c9 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -33,6 +33,7 @@
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/elog.h"
#include "utils/ps_status.h"
@@ -958,6 +959,36 @@ sendDir(const char *path, int basepathlen, bool sizeonly,
List *tablespaces,
char pathbuf[MAXPGPATH * 2];
struct stat statbuf;
int64 size = 0;
+ HTAB *unloggedHash = NULL; /* Unlogged tables in this
path. */
+ const char *unloggedDelim; /* Split this path from
parent path. */
+
+ /*
+ * Find any unlogged relations in this path and store them in a hash.
All
+ * unlogged relation forks except init will be excluded from the backup.
+ *
+ * Start by finding the location of the delimiter between the parent
+ * path and the current path.
+ */
+ unloggedDelim = strrchr(path, '/');
+
+ /* Does this path look like a database path (i.e. all digits)? */
+ if (unloggedDelim != NULL &&
+ strspn(unloggedDelim + 1, "0123456789") == strlen(unloggedDelim
+ 1))
+ {
+ /* Part of path that contains the parent directory. */
+ int parentPathLen = unloggedDelim - path;
+
+ /*
+ * Build the unlogged relation hash if the parent path is either
+ * $PGDATA/base or a tablespace version path.
+ */
+ if (strncmp(path, "./base", parentPathLen) == 0 ||
+ (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY)
- 1) &&
+ strncmp(unloggedDelim -
(sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
+ TABLESPACE_VERSION_DIRECTORY,
+ sizeof(TABLESPACE_VERSION_DIRECTORY) -
1) == 0))
+ unloggedHash = ResetUnloggedRelationsHash(path);
+ }
dir = AllocateDir(path);
while ((de = ReadDir(dir, path)) != NULL)
@@ -1007,6 +1038,15 @@ sendDir(const char *path, int basepathlen, bool
sizeonly, List *tablespaces,
if (excludeFound)
continue;
+ /* Exclude all forks for unlogged tables except the init fork.
*/
+ if (unloggedHash && ResetUnloggedRelationsMatch(
+ unloggedHash, de->d_name) == unloggedOther)
+ {
+ elog(DEBUG2, "unlogged relation file \"%s\" excluded
from backup",
+ de->d_name);
+ continue;
+ }
+
snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
/* Skip pg_control here to back up it last */
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index cdf4f5be37..4a22c4a657 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -4,7 +4,7 @@ use Cwd;
use Config;
use PostgresNode;
use TestLib;
-use Test::More tests => 79;
+use Test::More tests => 87;
program_help_ok('pg_basebackup');
program_version_ok('pg_basebackup');
@@ -66,6 +66,16 @@ foreach my $filename (
# positive.
$node->safe_psql('postgres', 'SELECT 1;');
+# Create an unlogged table to test that forks other than init are not copied.
+$node->safe_psql('postgres', 'CREATE UNLOGGED TABLE base_unlogged (id int)');
+
+my $baseUnloggedPath = $node->safe_psql('postgres',
+ q{select pg_relation_filepath('base_unlogged')});
+
+# Make sure main and init forks exist
+ok(-f "$pgdata/${baseUnloggedPath}_init", 'unlogged init fork in base');
+ok(-f "$pgdata/$baseUnloggedPath", 'unlogged main fork in base');
+
$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup", '-X', 'none' ],
'pg_basebackup runs');
ok(-f "$tempdir/backup/PG_VERSION", 'backup was created');
@@ -96,6 +106,12 @@ foreach my $filename (
ok(!-f "$tempdir/backup/$filename", "$filename not copied");
}
+# Unlogged relation forks other than init should not be copied
+ok(-f "$tempdir/backup/${baseUnloggedPath}_init",
+ 'unlogged init fork in backup');
+ok(!-f "$tempdir/backup/$baseUnloggedPath",
+ 'unlogged main fork not in backup');
+
# Make sure existing backup_label was ignored.
isnt(slurp_file("$tempdir/backup/backup_label"),
'DONOTCOPY', 'existing backup_label not copied');
@@ -147,7 +163,7 @@ unlink "$pgdata/$superlongname";
# skip on Windows.
SKIP:
{
- skip "symlinks not supported on Windows", 11 if ($windows_os);
+ skip "symlinks not supported on Windows", 15 if ($windows_os);
# Move pg_replslot out of $pgdata and create a symlink to it.
$node->stop;
@@ -177,6 +193,19 @@ SKIP:
my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";
is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');
+ # Create an unlogged table to test that forks other than init are not
copied.
+ $node->safe_psql('postgres',
+ 'CREATE UNLOGGED TABLE tblspc1_unlogged (id int) TABLESPACE
tblspc1;');
+
+ my $tblspc1UnloggedPath = $node->safe_psql(
+ 'postgres', q{select pg_relation_filepath('tblspc1_unlogged')});
+
+ # Make sure main and init forks exist
+ ok(-f "$pgdata/${tblspc1UnloggedPath}_init",
+ 'unlogged init fork in tablespace');
+ ok(-f "$pgdata/$tblspc1UnloggedPath",
+ 'unlogged main fork in tablespace');
+
$node->command_fails(
[ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ],
'plain format with tablespaces fails without tablespace
mapping');
@@ -195,11 +224,20 @@ SKIP:
"tablespace symlink was updated");
closedir $dh;
+ # Unlogged relation forks other than init should not be copied
+ my ($tblspc1UnloggedBackupPath) = $tblspc1UnloggedPath =~
/[^\/]*\/[^\/]*\/[^\/]*$/g;
+
+ ok(-f "$tempdir/tbackup/tblspc1/${tblspc1UnloggedBackupPath}_init",
+ 'unlogged init fork in tablespace backup');
+ ok(!-f "$tempdir/tbackup/tblspc1/$tblspc1UnloggedBackupPath",
+ 'unlogged imain fork not in tablespace backup');
+
ok( -d "$tempdir/backup1/pg_replslot",
'pg_replslot symlink copied as directory');
mkdir "$tempdir/tbl=spc2";
$node->safe_psql('postgres', "DROP TABLE test1;");
+ $node->safe_psql('postgres', "DROP TABLE tblspc1_unlogged;");
$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");
$node->safe_psql('postgres',
"CREATE TABLESPACE tblspc2 LOCATION
'$shorter_tempdir/tbl=spc2';");
