We have a huge amount of WAL files at backup site. A listing of the
directory takes several seconds. During startup pg_receivewal checks size
of all theus files. It does not check file integrity or gaps between
files. It takes several hours for our setup.
I have add an options that skip this file size checking. Default behavior
remains the same.
A patch looks huge due to large code block ident. Actually it consists of
option add and one if-branch.
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index d0a4079d50..6b8cf8fd58 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -51,6 +51,7 @@ static bool slot_exists_ok = false;
static bool do_drop_slot = false;
static bool do_sync = true;
static bool synchronous = false;
+static bool skip_filesize_check = false;
static char *replication_slot = NULL;
static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE;
static XLogRecPtr endpos = InvalidXLogRecPtr;
@@ -103,6 +104,7 @@ usage(void)
printf(_("\nOptional actions:\n"));
printf(_(" --create-slot create a new replication slot (for the slot's name see --slot)\n"));
printf(_(" --drop-slot drop the replication slot (for the slot's name see --slot)\n"));
+ printf(_(" --skip-filesize-check Don't check file sizes at directory specified by -D option \n"));
printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
@@ -291,169 +293,172 @@ FindStreamingStart(uint32 *tli)
*/
XLogFromFileName(dirent->d_name, &tli, &segno, WalSegSz);
- /*
- * Check that the segment has the right size, if it's supposed to be
- * completed. For non-compressed segments just check the on-disk size
- * and see if it matches a completed segment. For gzip-compressed
- * segments, look at the last 4 bytes of the compressed file, which is
- * where the uncompressed size is located for files with a size lower
- * than 4GB, and then compare it to the size of a completed segment.
- * The 4 last bytes correspond to the ISIZE member according to
- * http://www.zlib.org/rfc-gzip.html.
- *
- * For LZ4-compressed segments, uncompress the file in a throw-away
- * buffer keeping track of the uncompressed size, then compare it to
- * the size of a completed segment. Per its protocol, LZ4 does not
- * store the uncompressed size of an object by default. contentSize
- * is one possible way to do that, but we need to rely on a method
- * where WAL segments could have been compressed by a different source
- * than pg_receivewal, like an archive_command with lz4.
- */
- if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_NONE)
- {
- struct stat statbuf;
- char fullpath[MAXPGPATH * 2];
+ if(!skip_filesize_check)
+ {
+ /*
+ * Check that the segment has the right size, if it's supposed to be
+ * completed. For non-compressed segments just check the on-disk size
+ * and see if it matches a completed segment. For gzip-compressed
+ * segments, look at the last 4 bytes of the compressed file, which is
+ * where the uncompressed size is located for files with a size lower
+ * than 4GB, and then compare it to the size of a completed segment.
+ * The 4 last bytes correspond to the ISIZE member according to
+ * http://www.zlib.org/rfc-gzip.html.
+ *
+ * For LZ4-compressed segments, uncompress the file in a throw-away
+ * buffer keeping track of the uncompressed size, then compare it to
+ * the size of a completed segment. Per its protocol, LZ4 does not
+ * store the uncompressed size of an object by default. contentSize
+ * is one possible way to do that, but we need to rely on a method
+ * where WAL segments could have been compressed by a different source
+ * than pg_receivewal, like an archive_command with lz4.
+ */
+ if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_NONE)
+ {
+ struct stat statbuf;
+ char fullpath[MAXPGPATH * 2];
- snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
- if (stat(fullpath, &statbuf) != 0)
- pg_fatal("could not stat file \"%s\": %m", fullpath);
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
+ if (stat(fullpath, &statbuf) != 0)
+ pg_fatal("could not stat file \"%s\": %m", fullpath);
- if (statbuf.st_size != WalSegSz)
- {
- pg_log_warning("segment file \"%s\" has incorrect size %lld, skipping",
- dirent->d_name, (long long int) statbuf.st_size);
- continue;
+ if (statbuf.st_size != WalSegSz)
+ {
+ pg_log_warning("segment file \"%s\" has incorrect size %lld, skipping",
+ dirent->d_name, (long long int) statbuf.st_size);
+ continue;
+ }
}
- }
- else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_GZIP)
- {
- int fd;
- char buf[4];
- int bytes_out;
- char fullpath[MAXPGPATH * 2];
- int r;
-
- snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
-
- fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0)
- pg_fatal("could not open compressed file \"%s\": %m",
- fullpath);
- if (lseek(fd, (off_t) (-4), SEEK_END) < 0)
- pg_fatal("could not seek in compressed file \"%s\": %m",
- fullpath);
- r = read(fd, (char *) buf, sizeof(buf));
- if (r != sizeof(buf))
+ else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_GZIP)
{
- if (r < 0)
- pg_fatal("could not read compressed file \"%s\": %m",
+ int fd;
+ char buf[4];
+ int bytes_out;
+ char fullpath[MAXPGPATH * 2];
+ int r;
+
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
+
+ fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
+ if (fd < 0)
+ pg_fatal("could not open compressed file \"%s\": %m",
fullpath);
- else
- pg_fatal("could not read compressed file \"%s\": read %d of %zu",
- fullpath, r, sizeof(buf));
- }
+ if (lseek(fd, (off_t) (-4), SEEK_END) < 0)
+ pg_fatal("could not seek in compressed file \"%s\": %m",
+ fullpath);
+ r = read(fd, (char *) buf, sizeof(buf));
+ if (r != sizeof(buf))
+ {
+ if (r < 0)
+ pg_fatal("could not read compressed file \"%s\": %m",
+ fullpath);
+ else
+ pg_fatal("could not read compressed file \"%s\": read %d of %zu",
+ fullpath, r, sizeof(buf));
+ }
- close(fd);
- bytes_out = (buf[3] << 24) | (buf[2] << 16) |
- (buf[1] << 8) | buf[0];
+ close(fd);
+ bytes_out = (buf[3] << 24) | (buf[2] << 16) |
+ (buf[1] << 8) | buf[0];
- if (bytes_out != WalSegSz)
- {
- pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %d, skipping",
- dirent->d_name, bytes_out);
- continue;
+ if (bytes_out != WalSegSz)
+ {
+ pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %d, skipping",
+ dirent->d_name, bytes_out);
+ continue;
+ }
}
- }
- else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_LZ4)
- {
+ else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_LZ4)
+ {
#ifdef USE_LZ4
#define LZ4_CHUNK_SZ 64 * 1024 /* 64kB as maximum chunk size read */
- int fd;
- ssize_t r;
- size_t uncompressed_size = 0;
- char fullpath[MAXPGPATH * 2];
- char *outbuf;
- char *readbuf;
- LZ4F_decompressionContext_t ctx = NULL;
- LZ4F_decompressOptions_t dec_opt;
- LZ4F_errorCode_t status;
-
- memset(&dec_opt, 0, sizeof(dec_opt));
- snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
-
- fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0)
- pg_fatal("could not open file \"%s\": %m", fullpath);
-
- status = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
- if (LZ4F_isError(status))
- pg_fatal("could not create LZ4 decompression context: %s",
- LZ4F_getErrorName(status));
-
- outbuf = pg_malloc0(LZ4_CHUNK_SZ);
- readbuf = pg_malloc0(LZ4_CHUNK_SZ);
- do
- {
- char *readp;
- char *readend;
-
- r = read(fd, readbuf, LZ4_CHUNK_SZ);
- if (r < 0)
- pg_fatal("could not read file \"%s\": %m", fullpath);
-
- /* Done reading the file */
- if (r == 0)
- break;
-
- /* Process one chunk */
- readp = readbuf;
- readend = readbuf + r;
- while (readp < readend)
+ int fd;
+ ssize_t r;
+ size_t uncompressed_size = 0;
+ char fullpath[MAXPGPATH * 2];
+ char *outbuf;
+ char *readbuf;
+ LZ4F_decompressionContext_t ctx = NULL;
+ LZ4F_decompressOptions_t dec_opt;
+ LZ4F_errorCode_t status;
+
+ memset(&dec_opt, 0, sizeof(dec_opt));
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
+
+ fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
+ if (fd < 0)
+ pg_fatal("could not open file \"%s\": %m", fullpath);
+
+ status = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ pg_fatal("could not create LZ4 decompression context: %s",
+ LZ4F_getErrorName(status));
+
+ outbuf = pg_malloc0(LZ4_CHUNK_SZ);
+ readbuf = pg_malloc0(LZ4_CHUNK_SZ);
+ do
{
- size_t out_size = LZ4_CHUNK_SZ;
- size_t read_size = readend - readp;
-
- memset(outbuf, 0, LZ4_CHUNK_SZ);
- status = LZ4F_decompress(ctx, outbuf, &out_size,
- readp, &read_size, &dec_opt);
- if (LZ4F_isError(status))
- pg_fatal("could not decompress file \"%s\": %s",
- fullpath,
- LZ4F_getErrorName(status));
-
- readp += read_size;
- uncompressed_size += out_size;
+ char *readp;
+ char *readend;
+
+ r = read(fd, readbuf, LZ4_CHUNK_SZ);
+ if (r < 0)
+ pg_fatal("could not read file \"%s\": %m", fullpath);
+
+ /* Done reading the file */
+ if (r == 0)
+ break;
+
+ /* Process one chunk */
+ readp = readbuf;
+ readend = readbuf + r;
+ while (readp < readend)
+ {
+ size_t out_size = LZ4_CHUNK_SZ;
+ size_t read_size = readend - readp;
+
+ memset(outbuf, 0, LZ4_CHUNK_SZ);
+ status = LZ4F_decompress(ctx, outbuf, &out_size,
+ readp, &read_size, &dec_opt);
+ if (LZ4F_isError(status))
+ pg_fatal("could not decompress file \"%s\": %s",
+ fullpath,
+ LZ4F_getErrorName(status));
+
+ readp += read_size;
+ uncompressed_size += out_size;
+ }
+
+ /*
+ * No need to continue reading the file when the
+ * uncompressed_size exceeds WalSegSz, even if there are still
+ * data left to read. However, if uncompressed_size is equal
+ * to WalSegSz, it should verify that there is no more data to
+ * read.
+ */
+ } while (uncompressed_size <= WalSegSz && r > 0);
+
+ close(fd);
+ pg_free(outbuf);
+ pg_free(readbuf);
+
+ status = LZ4F_freeDecompressionContext(ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("could not free LZ4 decompression context: %s",
+ LZ4F_getErrorName(status));
+
+ if (uncompressed_size != WalSegSz)
+ {
+ pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %zu, skipping",
+ dirent->d_name, uncompressed_size);
+ continue;
}
-
- /*
- * No need to continue reading the file when the
- * uncompressed_size exceeds WalSegSz, even if there are still
- * data left to read. However, if uncompressed_size is equal
- * to WalSegSz, it should verify that there is no more data to
- * read.
- */
- } while (uncompressed_size <= WalSegSz && r > 0);
-
- close(fd);
- pg_free(outbuf);
- pg_free(readbuf);
-
- status = LZ4F_freeDecompressionContext(ctx);
- if (LZ4F_isError(status))
- pg_fatal("could not free LZ4 decompression context: %s",
- LZ4F_getErrorName(status));
-
- if (uncompressed_size != WalSegSz)
- {
- pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %zu, skipping",
- dirent->d_name, uncompressed_size);
- continue;
- }
#else
- pg_log_error("cannot check file \"%s\": compression with %s not supported by this build",
- dirent->d_name, "LZ4");
- exit(1);
+ pg_log_error("cannot check file \"%s\": compression with %s not supported by this build",
+ dirent->d_name, "LZ4");
+ exit(1);
#endif
+ }
}
/* Looks like a valid segment. Remember that we saw it. */
@@ -645,6 +650,7 @@ main(int argc, char **argv)
{"if-not-exists", no_argument, NULL, 3},
{"synchronous", no_argument, NULL, 4},
{"no-sync", no_argument, NULL, 5},
+ {"skip-filesize-check", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -743,6 +749,9 @@ main(int argc, char **argv)
case 5:
do_sync = false;
break;
+ case 6:
+ skip_filesize_check = true;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
