From 37c588047895af6b2c12b6c260a902aa83f42afc Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sat, 27 Feb 2021 09:03:50 +0500 Subject: [PATCH] Use different compression methods for FPIs --- src/backend/Makefile | 2 +- src/backend/access/transam/xlog.c | 1 + src/backend/access/transam/xloginsert.c | 65 +++++++++++++++++++++++-- src/backend/access/transam/xlogreader.c | 57 +++++++++++++++++++++- src/backend/utils/misc/guc.c | 17 +++++++ src/include/access/xlog.h | 1 + src/include/access/xlog_internal.h | 7 +++ src/include/access/xlogreader.h | 1 + src/include/access/xlogrecord.h | 7 +-- 9 files changed, 147 insertions(+), 11 deletions(-) diff --git a/src/backend/Makefile b/src/backend/Makefile index 9672e2cb43..de1cbac4c1 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -48,7 +48,7 @@ OBJS = \ LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) # The backend doesn't need everything that's in LIBS, however -LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) +LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) ifeq ($(with_systemd),yes) LIBS += -lsystemd diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e0c37f73f3..e11654bdd6 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -110,6 +110,7 @@ int CommitDelay = 0; /* precommit delay in microseconds */ int CommitSiblings = 5; /* # concurrent xacts needed to sleep */ int wal_retrieve_retry_interval = 5000; int max_slot_wal_keep_size_mb = -1; +int wal_compression_method = PGLZ_COMPRESSION_ID; #ifdef WAL_DEBUG bool XLOG_DEBUG = false; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 7052dc245e..a65823287e 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -33,6 +33,14 @@ #include "storage/proc.h" #include "utils/memutils.h" +#ifdef HAVE_LIBLZ4 +#include "lz4.h" +#endif + +#ifdef HAVE_LIBZ +#include +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -113,7 +121,8 @@ static XLogRecData *XLogRecordAssemble(RmgrId rmid, uint8 info, XLogRecPtr RedoRecPtr, bool doPageWrites, XLogRecPtr *fpw_lsn, int *num_fpi); static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, - uint16 hole_length, char *dest, uint16 *dlen); + uint16 hole_length, char *dest, + uint16 *dlen, CompressionId compression); /* * Begin constructing a WAL record. This must be called before the @@ -630,11 +639,20 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, */ if (wal_compression) { + bimg.compression_method = PGLZ_COMPRESSION_ID; +#ifdef HAVE_LIBLZ4 + if (wal_compression_method == LZ4_COMPRESSION_ID) + bimg.compression_method = wal_compression_method; +#endif +#ifdef HAVE_LIBZ + if (wal_compression_method == ZLIB_COMPRESSION_ID) + bimg.compression_method = wal_compression_method; +#endif is_compressed = XLogCompressBackupBlock(page, bimg.hole_offset, cbimg.hole_length, regbuf->compressed_page, - &compressed_len); + &compressed_len, bimg.compression_method); } /* @@ -827,7 +845,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, */ static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, - char *dest, uint16 *dlen) + char *dest, uint16 *dlen, CompressionId compression) { int32 orig_len = BLCKSZ - hole_length; int32 len; @@ -853,12 +871,49 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, else source = page; + if (compression == LZ4_COMPRESSION_ID) + { +#ifndef HAVE_LIBLZ4 + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("not built with lz4 support"))); +#else + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); +#endif + } + else if (compression == ZLIB_COMPRESSION_ID) + { + unsigned long len_l = PGLZ_MAX_BLCKSZ; +#ifndef HAVE_LIBZ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("not built with zlib support"))); +#else + if (compress((Bytef*)dest, &len_l, (Bytef*)source, orig_len) != Z_OK) + { + Assert(0); /* Zlib is not expected to fail compression */ + len_l = -1; + } + len = len_l; +#endif + } + else if (compression == PGLZ_COMPRESSION_ID) + { + len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unknown compression method requested"))); + } + + /* - * We recheck the actual size even if pglz_compress() reports success and + * We recheck the actual size even if compression reports success and * see if the number of bytes saved by compression is larger than the * length of extra data needed for the compressed version of block image. */ - len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); if (len >= 0 && len + extra_bytes < orig_len) { diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index bb95e0e527..00336621db 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -33,6 +33,14 @@ #include "utils/memutils.h" #endif +#ifdef HAVE_LIBLZ4 +#include "lz4.h" +#endif + +#ifdef HAVE_LIBZ +#include +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1291,6 +1299,7 @@ DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, char **errormsg) { COPY_HEADER_FIELD(&blk->bimg_len, sizeof(uint16)); COPY_HEADER_FIELD(&blk->hole_offset, sizeof(uint16)); + COPY_HEADER_FIELD(&blk->compression_method, sizeof(uint8)); COPY_HEADER_FIELD(&blk->bimg_info, sizeof(uint8)); blk->apply_image = ((blk->bimg_info & BKPIMAGE_APPLY) != 0); @@ -1565,8 +1574,52 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) if (bkpb->bimg_info & BKPIMAGE_IS_COMPRESSED) { /* If a backup block image is compressed, decompress it */ - if (pglz_decompress(ptr, bkpb->bimg_len, tmp.data, - BLCKSZ - bkpb->hole_length, true) < 0) + int32 decomp_result = -1; + if (bkpb->compression_method == PGLZ_COMPRESSION_ID) + { + decomp_result = pglz_decompress(ptr, bkpb->bimg_len, tmp.data, + BLCKSZ - bkpb->hole_length, true); + } + else if (bkpb->compression_method == LZ4_COMPRESSION_ID) + { +#ifndef HAVE_LIBLZ4 + report_invalid_record(record, "image at %X/%X is compressed with " + "lz4 not compiled into this build, block %d", + (uint32) (record->ReadRecPtr >> 32), + (uint32) record->ReadRecPtr, + block_id); + return false; +#else + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); +#endif + } + else if (bkpb->compression_method == ZLIB_COMPRESSION_ID) + { + unsigned long decomp_result_l = 0; +#ifndef HAVE_LIBZ + report_invalid_record(record, "image at %X/%X is compressed with " + "zlib not compiled into this build, block %d", + (uint32) (record->ReadRecPtr >> 32), + (uint32) record->ReadRecPtr, + block_id); + return false; +#else + decomp_result_l = BLCKSZ - bkpb->hole_length; + if (uncompress((Bytef*)tmp.data, &decomp_result_l, (Bytef*)ptr, bkpb->bimg_len) == Z_OK) + decomp_result = decomp_result_l; + else + decomp_result = -1; +#endif + } + else + { + report_invalid_record(record, "image at %X/%X is compressed with unknown codec, block %d", + (uint32) (record->ReadRecPtr >> 32), + (uint32) record->ReadRecPtr, + block_id); + return false; + } + if ( decomp_result < 0) { report_invalid_record(record, "invalid compressed image at %X/%X, block %d", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 00018abb7d..a3d8ef8be4 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -484,6 +484,13 @@ const struct config_enum_entry ssl_protocol_versions_info[] = { {NULL, 0, false} }; +const struct config_enum_entry wal_compression_options[] = { + {"pglz", PGLZ_COMPRESSION_ID, false}, + {"lz4", LZ4_COMPRESSION_ID, false}, + {"zlib", ZLIB_COMPRESSION_ID, false}, + {NULL, 0, false} +}; + StaticAssertDecl(lengthof(ssl_protocol_versions_info) == (PG_TLS1_3_VERSION + 2), "array length mismatch"); @@ -4721,6 +4728,16 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"wal_compression_method", PGC_SUSET, WAL_SETTINGS, + gettext_noop("Set the method used to compress full page images in the WAL."), + NULL + }, + &wal_compression_method, + PGLZ_COMPRESSION_ID, wal_compression_options, + NULL, NULL, NULL + }, + { {"dynamic_shared_memory_type", PGC_POSTMASTER, RESOURCES_MEM, gettext_noop("Selects the dynamic shared memory implementation used."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 75ec1073bd..6ca05e220f 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -175,6 +175,7 @@ typedef enum RecoveryState } RecoveryState; extern PGDLLIMPORT int wal_level; +extern PGDLLIMPORT int wal_compression_method; /* Is WAL archiving enabled (always or only while server is running normally)? */ #define XLogArchivingActive() \ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 224cae0246..2dfab59793 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -324,4 +324,11 @@ extern bool InArchiveRecovery; extern bool StandbyMode; extern char *recoveryRestoreCommand; +typedef enum CompressionId +{ + PGLZ_COMPRESSION_ID = 0, + LZ4_COMPRESSION_ID = 1, + ZLIB_COMPRESSION_ID = 2, +} CompressionId; + #endif /* XLOG_INTERNAL_H */ diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 21d200d3df..3d19c315d7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -133,6 +133,7 @@ typedef struct bool apply_image; /* has image that should be restored */ char *bkp_image; uint16 hole_offset; + uint8 compression_method; uint16 hole_length; uint16 bimg_len; uint8 bimg_info; diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 80c92a2498..3a885dc0c2 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -129,9 +129,10 @@ typedef struct XLogRecordBlockHeader */ typedef struct XLogRecordBlockImageHeader { - uint16 length; /* number of page image bytes */ - uint16 hole_offset; /* number of bytes before "hole" */ - uint8 bimg_info; /* flag bits, see below */ + uint16 length; /* number of page image bytes */ + uint16 hole_offset; /* number of bytes before "hole" */ + uint8 compression_method; /* compression method used for image */ + uint8 bimg_info; /* flag bits, see below */ /* * If BKPIMAGE_HAS_HOLE and BKPIMAGE_IS_COMPRESSED, an -- 2.24.3 (Apple Git-128)