On Sun, Dec 1, 2019 at 12:03 PM Michael Paquier <mich...@paquier.xyz> wrote:
>
> On Fri, Nov 01, 2019 at 09:38:37AM +0900, Moon, Insung wrote:
> > Of course, I may not have written the excellent quality code
> > correctly, so I will make an interim report if possible.
>
> The last patch has rotten, and does not apply anymore. A rebase would
> be nice, so I am switching the patch as waiting on author, and moved
> it to next CF.
>
We have discussed in off-list and weekly voice meeting for several
months that the purpose of this feature and the target for PG13 and we
concluded to step back and to focus on only internal key management
system for PG13. Transparent data encryption support is now the target
for PG14 or later. Key management system is an important
infrastructure for TDE but it can work independent of TDE. The plan
for PG13 we discussed is to introduce the internal key management
system that has one encryption key for whole database cluster and make
it have some interface to get encryption keys that are managed inside
PostgreSQL database in order to integrate it with other components
such as pgcrypto.
Idea is to get something encrypted and decrypted without ever knowing
the actual key that was used to encrypt it. The attached patch has two
APIs to wrap and unwrap the secret by the encryption key stored inside
database cluster. user generate a secret key locally and send it to
PostgreSQL server to wrap it using by pg_kmgr_wrap() and save it
somewhere. Then user can use the saved and wrapped secret key to
encrypt and decrypt user data by something like:
INSERT INTO tbl VALUES (pg_encrypt('user data', pg_kmgr_unwrap('xxxxx'));
Where 'xxxxx' is the result of pg_kmgr_wrap function.
I've attached the KMS patch. It requires openssl library. What the
patch does is simple and is not changed much from the previous version
patch that includes KMS and TDE; we generate one encryption key called
master key for whole database cluster at initdb time, which is stored
in pg_control and wrapped by key encryption key(KEK) derived from
user-provided passphrase. When postmaster starts up it verifies the
correctness of passphrase provided by user using hmac key which is
also derived from user-provided passphrase. The server won't start if
the passphrase is incorrect. Once the passphrase is verified the
master key is loaded to the shared buffer and is active.
I added two options to initdb: --cluster-passphrase-command and -e
that takes a passphrase command and a cipher algorithm (currently only
aes-128 and aes-256) respectively. The internal KMS is enabled by
executing initdb with those options as follows:
$ initdb -D data --cluster-passphrase-command="echo 'password'" -e aes-256
I believe the internal KMS would be useful several use cases but I'd
like to have discussion around the integration with pgcrypto because
pgcrypto would be the first user of the KMS and pgcrypto can be more
powerful with the KMS. I'll register this KMS patch to the next Commit
Fest.
I really appreciate peoples (CC-ing) who participated in off-list
discussion/meeting for many inputs, suggestions and reviewing codes.
Regards,
--
Masahiko Sawada http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
diff --git a/configure b/configure
index 3d9bd0bdf8..19f927cfb8 100755
--- a/configure
+++ b/configure
@@ -12111,7 +12111,7 @@ done
# defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it
# doesn't have these OpenSSL 1.1.0 functions. So check for individual
# functions.
- for ac_func in OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data
+ for ac_func in OPENSSL_init_ssl OPENSSL_init_crypto BIO_get_data BIO_meth_new ASN1_STRING_get0_data
do :
as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
diff --git a/configure.in b/configure.in
index 34aea0b4ac..0a94ea37c5 100644
--- a/configure.in
+++ b/configure.in
@@ -1193,7 +1193,7 @@ if test "$with_openssl" = yes ; then
# defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it
# doesn't have these OpenSSL 1.1.0 functions. So check for individual
# functions.
- AC_CHECK_FUNCS([OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data])
+ AC_CHECK_FUNCS([OPENSSL_init_ssl OPENSSL_init_crypto BIO_get_data BIO_meth_new ASN1_STRING_get0_data])
# OpenSSL versions before 1.1.0 required setting callback functions, for
# thread-safety. In 1.1.0, it's no longer required, and CRYPTO_lock()
# function was removed.
diff --git a/src/backend/Makefile b/src/backend/Makefile
index b0d2be7ee0..64ce50474f 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -21,7 +21,7 @@ SUBDIRS = access bootstrap catalog parser commands executor foreign lib libpq \
main nodes optimizer partitioning port postmaster \
regex replication rewrite \
statistics storage tcop tsearch utils $(top_builddir)/src/timezone \
- jit
+ jit crypto
include $(srcdir)/common.mk
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6bc1a6b46d..e27f771be2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -41,6 +41,7 @@
#include "catalog/pg_database.h"
#include "commands/tablespace.h"
#include "common/controldata_utils.h"
+#include "crypto/kmgr.h"
#include "miscadmin.h"
#include "pg_trace.h"
#include "pgstat.h"
@@ -77,6 +78,7 @@
#include "utils/timestamp.h"
extern uint32 bootstrap_data_checksum_version;
+extern uint32 bootstrap_data_encryption_cipher;
/* Unsupported old recovery command file names (relative to $PGDATA) */
#define RECOVERY_COMMAND_FILE "recovery.conf"
@@ -4780,6 +4782,10 @@ ReadControlFile(void)
/* Make the initdb settings visible as GUC variables, too */
SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no",
PGC_INTERNAL, PGC_S_OVERRIDE);
+
+ SetConfigOption("data_encryption_cipher",
+ kmgr_cipher_string(GetDataEncryptionCipher()),
+ PGC_INTERNAL, PGC_S_OVERRIDE);
}
/*
@@ -4812,6 +4818,13 @@ GetMockAuthenticationNonce(void)
return ControlFile->mock_authentication_nonce;
}
+WrappedEncKeyWithHmac *
+GetMasterEncryptionKey(void)
+{
+ Assert(ControlFile != NULL);
+ return &(ControlFile->master_dek);
+}
+
/*
* Are checksums enabled for data pages?
*/
@@ -4822,6 +4835,13 @@ DataChecksumsEnabled(void)
return (ControlFile->data_checksum_version > 0);
}
+int
+GetDataEncryptionCipher(void)
+{
+ Assert(ControlFile != NULL);
+ return ControlFile->data_encryption_cipher;
+}
+
/*
* Returns a fake LSN for unlogged relations.
*
@@ -5088,6 +5108,7 @@ BootStrapXLOG(void)
XLogPageHeader page;
XLogLongPageHeader longpage;
XLogRecord *record;
+ WrappedEncKeyWithHmac *masterkey;
char *recptr;
bool use_existent;
uint64 sysidentifier;
@@ -5165,6 +5186,12 @@ BootStrapXLOG(void)
SetMultiXactIdLimit(checkPoint.oldestMulti, checkPoint.oldestMultiDB, true);
SetCommitTsLimit(InvalidTransactionId, InvalidTransactionId);
+ /*
+ * Bootstrap key management module beforehand in order to encrypt the first
+ * xlog record.
+ */
+ masterkey = BootStrapKmgr(bootstrap_data_encryption_cipher);
+
/* Set up the XLOG page header */
page->xlp_magic = XLOG_PAGE_MAGIC;
page->xlp_info = XLP_LONG_HEADER;
@@ -5240,6 +5267,9 @@ BootStrapXLOG(void)
ControlFile->checkPoint = checkPoint.redo;
ControlFile->checkPointCopy = checkPoint;
ControlFile->unloggedLSN = FirstNormalUnloggedLSN;
+ if (masterkey)
+ memcpy(&(ControlFile->master_dek), masterkey,
+ sizeof(WrappedEncKeyWithHmac));
/* Set important parameter values for use when replaying WAL */
ControlFile->MaxConnections = MaxConnections;
@@ -5251,6 +5281,7 @@ BootStrapXLOG(void)
ControlFile->wal_log_hints = wal_log_hints;
ControlFile->track_commit_timestamp = track_commit_timestamp;
ControlFile->data_checksum_version = bootstrap_data_checksum_version;
+ ControlFile->data_encryption_cipher = bootstrap_data_encryption_cipher;
/* some additional ControlFile fields are set in WriteControlFile() */
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index 8ea033610d..10ef1f1260 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -28,6 +28,7 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
#include "common/link-canary.h"
+#include "crypto/kmgr.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -52,6 +53,9 @@
uint32 bootstrap_data_checksum_version = 0; /* No checksum */
+/* No encryption */
+uint32 bootstrap_data_encryption_cipher = KMGR_ENCRYPTION_OFF;
+
#define ALLOC(t, c) \
((t *) MemoryContextAllocZero(TopMemoryContext, (unsigned)(c) * sizeof(t)))
@@ -226,7 +230,7 @@ AuxiliaryProcessMain(int argc, char *argv[])
/* If no -x argument, we are a CheckerProcess */
MyAuxProcType = CheckerProcess;
- while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:X:-:")) != -1)
+ while ((flag = getopt(argc, argv, "B:c:d:D:e:Fkr:x:X:-:")) != -1)
{
switch (flag)
{
@@ -249,6 +253,10 @@ AuxiliaryProcessMain(int argc, char *argv[])
pfree(debugstr);
}
break;
+
+ case 'e':
+ bootstrap_data_encryption_cipher = kmgr_cipher_value(optarg);
+ break;
case 'F':
SetConfigOption("fsync", "false", PGC_POSTMASTER, PGC_S_ARGV);
break;
diff --git a/src/backend/crypto/Makefile b/src/backend/crypto/Makefile
new file mode 100644
index 0000000000..d7bfa17b39
--- /dev/null
+++ b/src/backend/crypto/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+# Makefile for cryptographic
+#
+# IDENTIFICATION
+# src/backend/crypto/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/crypto
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = kmgr.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/crypto/kmgr.c b/src/backend/crypto/kmgr.c
new file mode 100644
index 0000000000..697c387864
--- /dev/null
+++ b/src/backend/crypto/kmgr.c
@@ -0,0 +1,272 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.c
+ * Encryption key management module.
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/storage/encryption/kmgr.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <unistd.h>
+
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+
+#include "access/xlog.h"
+#include "common/sha2.h"
+#include "common/kmgr_utils.h"
+#include "crypto/kmgr.h"
+#include "storage/fd.h"
+#include "storage/shmem.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+
+/*
+ * Key encryption key. This key is derived from the passphrase provided
+ * by user when startup. This variable is set during verification
+ * of user given passphrase. After verification, the plain key data
+ * is set to this variable.
+ */
+static uint8 keyEncKey[KMGR_KEK_LEN];
+
+/*
+ * Mater encryption key. Similar to key encryption key, this
+ * store the plain key data.
+ */
+static uint8 masterEncKey[KMGR_MAX_DEK_LEN];
+
+/* GUC variable */
+char *cluster_passphrase_command = NULL;
+
+int data_encryption_cipher;
+int EncryptionKeyLen;
+
+/*
+ * This function must be called ONCE on system install. We retrieve the KEK,
+ * generate the master key.
+ */
+WrappedEncKeyWithHmac *
+BootStrapKmgr(int bootstrap_data_encryption_cipher)
+{
+ WrappedEncKeyWithHmac *ret_mk;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ uint8 hmackey[KMGR_HMAC_KEY_LEN];
+ int passlen;
+
+ if (bootstrap_data_encryption_cipher == KMGR_ENCRYPTION_OFF)
+ return NULL;
+
+#ifndef USE_OPENSSL
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ (errmsg("cluster encryption is not supported because OpenSSL is not supported by this build"),
+ errhint("Compile with --with-openssl to use cluster encryption."))));
+#endif
+
+ ret_mk = palloc0(sizeof(WrappedEncKeyWithHmac));
+
+ /*
+ * Set data encryption cipher so that subsequent bootstrapping process
+ * can proceed.
+ */
+ SetConfigOption("data_encryption_cipher",
+ kmgr_cipher_string(bootstrap_data_encryption_cipher),
+ PGC_INTERNAL, PGC_S_OVERRIDE);
+
+ /* Get key encryption key from command */
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase, KMGR_MAX_PASSPHRASE_LEN);
+ if (passlen < KMGR_MIN_PASSPHRASE_LEN)
+ ereport(ERROR,
+ (errmsg("passphrase must be more than %d bytes",
+ KMGR_MIN_PASSPHRASE_LEN)));
+
+ /* Get key encryption key and HMAC key from passphrase */
+ kmgr_derive_keys(passphrase, passlen, keyEncKey, hmackey);
+
+ /* Generate the master encryption key */
+ if (!pg_strong_random(masterEncKey, EncryptionKeyLen))
+ ereport(ERROR,
+ (errmsg("failed to generate cluster encryption key")));
+
+ /* Wrap the master key by KEK */
+ kmgr_wrap_key(keyEncKey, masterEncKey, EncryptionKeyLen, ret_mk->key);
+
+ /* Compute HMAC of the master key */
+ kmgr_compute_HMAC(hmackey, ret_mk->key, SizeOfWrappedDEK(),
+ ret_mk->hmac);
+
+ /* return keys and HMACs generated during bootstrap */
+ return ret_mk;
+}
+
+/*
+ * Get encryption key passphrase and verify it, then get the un-wrapped
+ * master encryption key. This function is called by postmaster at startup time.
+ */
+void
+InitializeKmgr(void)
+{
+ WrappedEncKeyWithHmac *wrapped_mk;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ int passlen;
+
+ if (!DataEncryptionEnabled())
+ return;
+
+ /* Get cluster passphrase */
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase, KMGR_MAX_PASSPHRASE_LEN);
+
+ /* Get two wrapped keys stored in control file */
+ wrapped_mk = GetMasterEncryptionKey();
+
+ /* Verify the correctness of given passphrase */
+ if (!kmgr_verify_passphrase(passphrase, passlen, wrapped_mk,
+ SizeOfWrappedDEK()))
+ ereport(ERROR,
+ (errmsg("cluster passphrase does not match expected passphrase")));
+
+ kmgr_derive_keys(passphrase, passlen, keyEncKey, NULL);
+
+ /* The passphrase is correct, unwrap the master key */
+ kmgr_unwrap_key(keyEncKey, wrapped_mk->key, SizeOfWrappedDEK(),
+ masterEncKey);
+}
+
+/* Return plain cluster encryption key */
+const char *
+KmgrGetMasterEncryptionKey(void)
+{
+ return DataEncryptionEnabled() ?
+ pstrdup((const char *) masterEncKey) : NULL;
+}
+
+extern
+void assign_data_encryption_cipher(int new_encryption_cipher,
+ void *extra)
+{
+ switch (new_encryption_cipher)
+ {
+ case KMGR_ENCRYPTION_OFF :
+ EncryptionKeyLen = 0;
+ break;
+ case KMGR_ENCRYPTION_AES128:
+ EncryptionKeyLen = 16;
+ break;
+ case KMGR_ENCRYPTION_AES256:
+ EncryptionKeyLen = 32;
+ break;
+ }
+}
+
+/*
+ * SQL function to rotate the cluster encryption key. This function
+ * assumes that the cluster_passphrase_command is already reloaded
+ * to the new value.
+ */
+Datum
+pg_rotate_encryption_key(PG_FUNCTION_ARGS)
+{
+ WrappedEncKeyWithHmac new_masterkey;
+ WrappedEncKeyWithHmac *cur_masterkey;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ uint8 new_kek[KMGR_KEK_LEN];
+ uint8 new_hmackey[KMGR_HMAC_KEY_LEN];
+ int passlen;
+
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase,
+ KMGR_MAX_PASSPHRASE_LEN);
+ if (passlen < KMGR_MIN_PASSPHRASE_LEN)
+ ereport(ERROR,
+ (errmsg("passphrase must be more than %d bytes",
+ KMGR_MIN_PASSPHRASE_LEN)));
+
+ kmgr_derive_keys(passphrase, passlen, new_kek, new_hmackey);
+
+ /* Copy the current master encryption key */
+ memcpy(&(new_masterkey.key), masterEncKey, EncryptionKeyLen);
+
+ /*
+ * Wrap and compute HMAC of the master key by the new key
+ * encryption key.
+ */
+ kmgr_wrap_key(new_kek, new_masterkey.key, EncryptionKeyLen,
+ new_masterkey.key);
+ kmgr_compute_HMAC(new_hmackey, new_masterkey.key,
+ SizeOfWrappedDEK(), new_masterkey.hmac);
+
+ /* Update control file */
+ LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+ cur_masterkey = GetMasterEncryptionKey();
+ memcpy(cur_masterkey, &new_masterkey, sizeof(WrappedEncKeyWithHmac));
+ UpdateControlFile();
+ LWLockRelease(ControlFileLock);
+
+ PG_RETURN_BOOL(true);
+}
+
+Datum
+pg_kmgr_wrap(PG_FUNCTION_ARGS)
+{
+ bytea *data = PG_GETARG_BYTEA_PP(0);
+ bytea *res;
+ unsigned datalen;
+ unsigned reslen;
+ bool ret;
+
+ datalen = VARSIZE_ANY_EXHDR(data);
+
+ if (datalen % 8 != 0)
+ ereport(ERROR,
+ (errmsg("input data must be multiple of 8 bytes")));
+
+ reslen = VARHDRSZ + datalen + AES256_KEY_WRAP_VALUE_LEN;
+ res = palloc(reslen);
+
+ ret = kmgr_wrap_key(keyEncKey, (uint8 *) VARDATA_ANY(data), datalen,
+ (uint8 *) VARDATA(res));
+ if (!ret)
+ ereport(ERROR,
+ (errmsg("could not wrap the given secret")));
+
+ SET_VARSIZE(res, reslen);
+ PG_RETURN_BYTEA_P(res);
+}
+
+Datum
+pg_kmgr_unwrap(PG_FUNCTION_ARGS)
+{
+ bytea *data = PG_GETARG_BYTEA_PP(0);
+ bytea *res;
+ unsigned datalen;
+ unsigned reslen;
+ bool ret;
+
+ datalen = VARSIZE_ANY_EXHDR(data);
+
+ if (datalen % 8 != 0)
+ ereport(ERROR,
+ (errmsg("input data must be multiple of 8 bytes")));
+
+ reslen = VARHDRSZ + datalen - AES256_KEY_WRAP_VALUE_LEN;
+ res = palloc(reslen);
+
+ ret = kmgr_unwrap_key(keyEncKey, (uint8 *) VARDATA_ANY(data), datalen,
+ (uint8 *) VARDATA(res));
+ if (!ret)
+ ereport(ERROR,
+ (errmsg("could not unwrap the given secret")));
+
+ SET_VARSIZE(res, reslen);
+ PG_RETURN_BYTEA_P(res);
+}
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index fabcf31de8..b8812c69d4 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3954,6 +3954,15 @@ pgstat_get_wait_io(WaitEventIO w)
case WAIT_EVENT_DSM_FILL_ZERO_WRITE:
event_name = "DSMFillZeroWrite";
break;
+ case WAIT_EVENT_KMGR_FILE_READ:
+ event_name = "KmgrFileRead";
+ break;
+ case WAIT_EVENT_KMGR_FILE_SYNC:
+ event_name = "KmgrFileSync";
+ break;
+ case WAIT_EVENT_KMGR_FILE_WRITE:
+ event_name = "KmgrFileWrite";
+ break;
case WAIT_EVENT_LOCK_FILE_ADDTODATADIR_READ:
event_name = "LockFileAddToDataDirRead";
break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 9ff2832c00..97448437d4 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -100,6 +100,7 @@
#include "common/file_perm.h"
#include "common/ip.h"
#include "common/string.h"
+#include "crypto/kmgr.h"
#include "lib/ilist.h"
#include "libpq/auth.h"
#include "libpq/libpq.h"
@@ -1335,6 +1336,11 @@ PostmasterMain(int argc, char *argv[])
*/
autovac_init();
+ /*
+ * Initialize cluster encryption key manager.
+ */
+ InitializeKmgr();
+
/*
* Load configuration files for client authentication.
*/
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 0b7bc1fd03..daf55df11a 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -42,6 +42,7 @@
#include "catalog/pg_type.h"
#include "commands/async.h"
#include "commands/prepare.h"
+#include "crypto/kmgr.h"
#include "executor/spi.h"
#include "jit/jit.h"
#include "libpq/libpq.h"
@@ -3898,6 +3899,13 @@ PostgresMain(int argc, char *argv[],
/* Early initialization */
BaseInit();
+ /*
+ * Initialize kmgr for cluster encryption. Since kmgr needs to attach to
+ * shared memory the initialization must be called after BaseInit().
+ */
+ if (!IsUnderPostmaster)
+ InitializeKmgr();
+
/*
* Create a per-backend PGPROC struct in shared memory, except in the
* EXEC_BACKEND case where this was done in SubPostmasterMain. We must do
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 8d951ce404..5ddaf578a3 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -43,6 +43,7 @@
#include "commands/vacuum.h"
#include "commands/variable.h"
#include "common/string.h"
+#include "crypto/kmgr.h"
#include "funcapi.h"
#include "jit/jit.h"
#include "libpq/auth.h"
@@ -73,6 +74,7 @@
#include "replication/walsender.h"
#include "storage/bufmgr.h"
#include "storage/dsm_impl.h"
+#include "storage/standby.h"
#include "storage/fd.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -460,6 +462,13 @@ const struct config_enum_entry ssl_protocol_versions_info[] = {
{NULL, 0, false}
};
+const struct config_enum_entry data_encryption_cipher_options[] = {
+ {"off", KMGR_ENCRYPTION_OFF, false},
+ {"aes-128", KMGR_ENCRYPTION_AES128, false},
+ {"aes-256", KMGR_ENCRYPTION_AES256, false},
+ {NULL, 0, false}
+};
+
static struct config_enum_entry shared_memory_options[] = {
#ifndef WIN32
{"sysv", SHMEM_TYPE_SYSV, false},
@@ -712,6 +721,8 @@ const char *const config_group_names[] =
gettext_noop("Statistics / Monitoring"),
/* STATS_COLLECTOR */
gettext_noop("Statistics / Query and Index Statistics Collector"),
+ /* ENCRYPTION */
+ gettext_noop("Encryption"),
/* AUTOVACUUM */
gettext_noop("Autovacuum"),
/* CLIENT_CONN */
@@ -4160,7 +4171,7 @@ static struct config_string ConfigureNamesString[] =
{"ssl_ciphers", PGC_SIGHUP, CONN_AUTH_SSL,
gettext_noop("Sets the list of allowed SSL ciphers."),
NULL,
- GUC_SUPERUSER_ONLY
+ GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
},
&SSLCipherSuites,
#ifdef USE_OPENSSL
@@ -4207,6 +4218,16 @@ static struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"cluster_passphrase_command", PGC_SIGHUP, ENCRYPTION,
+ gettext_noop("Command to obtain passphrase for database encryption."),
+ NULL
+ },
+ &cluster_passphrase_command,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"application_name", PGC_USERSET, LOGGING_WHAT,
gettext_noop("Sets the application name to be reported in statistics and logs."),
@@ -4600,6 +4621,19 @@ static struct config_enum ConfigureNamesEnum[] =
NULL, NULL, NULL
},
+ {
+ {"data_encryption_cipher", PGC_INTERNAL, PRESET_OPTIONS,
+ gettext_noop("Specify encryption algorithms to use."),
+ NULL,
+ GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE,
+ GUC_SUPERUSER_ONLY
+ },
+ &data_encryption_cipher,
+ KMGR_ENCRYPTION_OFF,
+ data_encryption_cipher_options,
+ NULL, assign_data_encryption_cipher, NULL
+ },
+
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 087190ce63..52f37901bd 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -613,6 +613,11 @@
# autovacuum, -1 means use
# vacuum_cost_limit
+#------------------------------------------------------------------------------
+# ENCRYPTION
+#------------------------------------------------------------------------------
+
+#cluster_passphrase_command = ''
#------------------------------------------------------------------------------
# CLIENT CONNECTION DEFAULTS
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 1f6d8939be..713b6abc51 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -114,6 +114,13 @@ static const char *const auth_methods_local[] = {
NULL
};
+static const char *const encryption_ciphers[] = {
+ "none",
+ "aes-128",
+ "aes-256",
+ NULL
+};
+
/*
* these values are passed in by makefile defines
*/
@@ -145,6 +152,8 @@ static bool data_checksums = false;
static char *xlog_dir = NULL;
static char *str_wal_segment_size_mb = NULL;
static int wal_segment_size_mb;
+static char *enc_cipher = NULL;
+static char *cluster_passphrase = NULL;
/* internal vars */
@@ -1208,6 +1217,13 @@ setup_config(void)
"password_encryption = scram-sha-256");
}
+ if (cluster_passphrase)
+ {
+ snprintf(repltok, sizeof(repltok), "cluster_passphrase_command = '%s'",
+ escape_quotes(cluster_passphrase));
+ conflines = replace_token(conflines, "#cluster_passphrase_command = ''", repltok);
+ }
+
/*
* If group access has been enabled for the cluster then it makes sense to
* ensure that the log files also allow group access. Otherwise a backup
@@ -1418,14 +1434,15 @@ bootstrap_template1(void)
unsetenv("PGCLIENTENCODING");
snprintf(cmd, sizeof(cmd),
- "\"%s\" --boot -x1 -X %u %s %s %s",
+ "\"%s\" --boot -x1 -X %u %s %s %s %s %s",
backend_exec,
wal_segment_size_mb * (1024 * 1024),
data_checksums ? "-k" : "",
+ enc_cipher ? "-e" : "",
+ enc_cipher ? enc_cipher : "",
boot_options,
debug ? "-d 5" : "");
-
PG_CMD_OPEN;
for (line = bki_lines; *line != NULL; line++)
@@ -2348,6 +2365,9 @@ usage(const char *progname)
printf(_(" --wal-segsize=SIZE size of WAL segments, in megabytes\n"));
printf(_("\nLess commonly used options:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
+ printf(_(" -e --enc-cipher=TYPE set encryption cipher (AES-128/AES-256) for data encryption\n"));
+ printf(_(" -c --cluster-passphrase-command=COMMAND\n"
+ " set command to obtain passphrase for data encryption key\n"));
printf(_(" -k, --data-checksums use data page checksums\n"));
printf(_(" -L DIRECTORY where to find the input files\n"));
printf(_(" -n, --no-clean do not clean up after errors\n"));
@@ -2413,6 +2433,42 @@ check_need_password(const char *authmethodlocal, const char *authmethodhost)
}
}
+static void
+check_encryption_cipher(const char *cipher, const char *passphrase,
+ const char *const *valid_ciphers)
+{
+ const char *const *p;
+
+ if (!cipher && !passphrase)
+ return;
+
+#ifndef USE_OPENSSL
+ pg_log_error("cluster encryption is not supported because OpenSSL is not supported by this build");
+ exit(1);
+#endif
+
+ /* Check both options must be specified at the same time */
+ if (cipher && !passphrase)
+ {
+ pg_log_error("encryption passphrase command must be specified when encryption cipher is specified");
+ exit(1);
+ }
+
+ if (!cipher && passphrase)
+ {
+ pg_log_error("encryption cipher must be specified when encryption passphrase command is specified");
+ exit(1);
+ }
+
+ for (p = valid_ciphers; *p; p++)
+ {
+ if (strcasecmp(cipher, *p) == 0)
+ return;
+ }
+
+ pg_log_error("invalid encryption cipher \"%s\"\nencryption cipher options are AES-128 and AES-256", cipher);
+ exit(1);
+}
void
setup_pgdata(void)
@@ -3026,6 +3082,8 @@ main(int argc, char *argv[])
{"wal-segsize", required_argument, NULL, 12},
{"data-checksums", no_argument, NULL, 'k'},
{"allow-group-access", no_argument, NULL, 'g'},
+ {"enc-cipher", required_argument, NULL, 'e'},
+ {"cluster-passphrase-command", required_argument, NULL, 'c'},
{NULL, 0, NULL, 0}
};
@@ -3067,7 +3125,7 @@ main(int argc, char *argv[])
/* process command-line options */
- while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "c:dD:E:e:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -3149,6 +3207,12 @@ main(int argc, char *argv[])
case 9:
pwfilename = pg_strdup(optarg);
break;
+ case 'e':
+ enc_cipher = pg_strdup(optarg);
+ break;
+ case 'c':
+ cluster_passphrase = pg_strdup(optarg);
+ break;
case 's':
show_setting = true;
break;
@@ -3227,6 +3291,8 @@ main(int argc, char *argv[])
check_need_password(authmethodlocal, authmethodhost);
+ check_encryption_cipher(enc_cipher, cluster_passphrase, encryption_ciphers);
+
/* set wal segment size */
if (str_wal_segment_size_mb == NULL)
wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
@@ -3286,6 +3352,11 @@ main(int argc, char *argv[])
else
printf(_("Data page checksums are disabled.\n"));
+ if (enc_cipher)
+ printf(_("Data encryption using %s is enabled.\n"), enc_cipher);
+ else
+ printf(_("Data encryption is disabled.\n"));
+
if (pwprompt || pwfilename)
get_su_pwd();
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index 971ae73f54..792a9e7565 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -100,6 +100,7 @@ static const char *const skip[] = {
"pg_control",
"pg_filenode.map",
"pg_internal.init",
+ "pg_kmgr",
"PG_VERSION",
#ifdef EXEC_BACKEND
"config_exec_params",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 19e21ab491..14d3c6757b 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -25,6 +25,7 @@
#include "access/xlog_internal.h"
#include "catalog/pg_control.h"
#include "common/controldata_utils.h"
+#include "common/kmgr_utils.h"
#include "common/logging.h"
#include "getopt_long.h"
#include "pg_getopt.h"
@@ -83,7 +84,6 @@ wal_level_str(WalLevel wal_level)
return _("unrecognized wal_level");
}
-
int
main(int argc, char *argv[])
{
@@ -333,5 +333,7 @@ main(int argc, char *argv[])
ControlFile->data_checksum_version);
printf(_("Mock authentication nonce: %s\n"),
mock_auth_nonce_str);
+ printf(_("Data encryption cipher: %s\n"),
+ kmgr_cipher_string(ControlFile->data_encryption_cipher));
return 0;
}
diff --git a/src/common/Makefile b/src/common/Makefile
index ffb0f6edff..c101703088 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -51,11 +51,13 @@ OBJS_COMMON = \
config_info.o \
controldata_utils.o \
d2s.o \
+ cipher.o \
exec.o \
f2s.o \
file_perm.o \
ip.o \
keywords.o \
+ kmgr_utils.o \
kwlookup.o \
link-canary.o \
md5.o \
@@ -73,7 +75,9 @@ OBJS_COMMON = \
wait_error.o
ifeq ($(with_openssl),yes)
-OBJS_COMMON += sha2_openssl.o
+OBJS_COMMON += \
+ sha2_openssl.o \
+ cipher_openssl.o
else
OBJS_COMMON += sha2.o
endif
diff --git a/src/common/cipher.c b/src/common/cipher.c
new file mode 100644
index 0000000000..8b8a76af2e
--- /dev/null
+++ b/src/common/cipher.c
@@ -0,0 +1,95 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.c
+ * Shared frontend/backend for cryptographic functions
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/cipher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+#include "common/cipher_openssl.h"
+
+pg_cipher_ctx *
+pg_cipher_ctx_create(void)
+{
+#ifdef USE_OPENSSL
+ return ossl_cipher_ctx_create();
+#endif
+ return NULL;
+}
+
+void
+pg_cipher_setup(void)
+{
+#ifdef USE_OPENSSL
+ ossl_cipher_setup();
+#endif
+}
+
+bool
+pg_aes256_ctr_wrap_init(pg_cipher_ctx *ctx)
+{
+#ifdef USE_OPENSSL
+ return ossl_aes256_ctr_wrap_init(ctx);
+#endif
+ return false;
+}
+
+bool
+pg_aes256_ctr_unwrap_init(pg_cipher_ctx *ctx)
+{
+#ifdef USE_OPENSSL
+ return ossl_aes256_ctr_unwrap_init(ctx);
+#endif
+ return false;
+}
+
+bool
+pg_cipher_encrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest, int *dest_size)
+
+{
+ bool r = true;
+#ifdef USE_OPENSSL
+ r = ossl_cipher_encrypt(ctx, key, input, input_size, iv,
+ dest, dest_size);
+#endif
+ return r;
+}
+
+bool
+pg_cipher_decrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest, int *dest_size)
+{
+ bool r = true;
+#ifdef USE_OPENSSL
+ r = ossl_cipher_decrypt(ctx, key, input, input_size, iv,
+ dest, dest_size);
+#endif
+ return r;
+}
+
+bool
+pg_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result, int *result_size)
+{
+ bool r = true;
+#ifdef USE_OPENSSL
+ r = ossl_compute_HMAC(key, data, data_size, result,
+ result_size);
+#endif
+ return r;
+}
diff --git a/src/common/cipher_openssl.c b/src/common/cipher_openssl.c
new file mode 100644
index 0000000000..519653c7d5
--- /dev/null
+++ b/src/common/cipher_openssl.c
@@ -0,0 +1,99 @@
+/*-------------------------------------------------------------------------
+ * cipher_openssl.c
+ * Cryptographic function using OpenSSL
+ *
+ * This contains the common low-level functions needed in both frontend and
+ * backend, for implement the database encryption.
+ *
+ * Portions Copyright (c) 2017-2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/cipher_openssl.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher_openssl.h"
+
+#include <openssl/conf.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/hmac.h>
+
+pg_cipher_ctx *
+ossl_cipher_ctx_create(void)
+{
+ return EVP_CIPHER_CTX_new();
+}
+
+bool
+ossl_cipher_setup(void)
+{
+#ifdef HAVE_OPENSSL_INIT_CRYPTO
+ /* Setup OpenSSL */
+ if (!OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL))
+ return false;
+ return true;
+#endif
+ return false;
+}
+
+bool
+ossl_aes256_ctr_wrap_init(pg_cipher_ctx *ctx)
+{
+ EVP_CIPHER_CTX_set_flags(ctx, EVP_CIPHER_CTX_FLAG_WRAP_ALLOW);
+ if (!EVP_EncryptInit_ex(ctx, EVP_aes_256_wrap(), NULL, NULL, NULL))
+ return false;
+
+ EVP_CIPHER_CTX_set_key_length(ctx, AES256_KEY_LEN);
+ return true;
+}
+
+bool
+ossl_aes256_ctr_unwrap_init(pg_cipher_ctx *ctx)
+{
+ EVP_CIPHER_CTX_set_flags(ctx, EVP_CIPHER_CTX_FLAG_WRAP_ALLOW);
+ if (!EVP_DecryptInit_ex(ctx, EVP_aes_256_wrap(), NULL, NULL, NULL))
+ return false;
+
+ EVP_CIPHER_CTX_set_key_length(ctx, AES256_KEY_LEN);
+ return true;
+}
+
+bool
+ossl_cipher_encrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size)
+{
+ if (!EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv))
+ return false;
+
+ return EVP_EncryptUpdate(ctx, dest, dest_size, input, input_size);
+}
+
+bool
+ossl_cipher_decrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size)
+{
+ if (!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
+ return false;
+
+ return EVP_DecryptUpdate(ctx, dest, dest_size, input, input_size);
+}
+
+bool
+ossl_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size)
+{
+ return HMAC(EVP_sha256(), key, AES256_KEY_LEN, data,
+ (uint32) data_size, result, (uint32 *) result_size);
+}
diff --git a/src/common/kmgr_utils.c b/src/common/kmgr_utils.c
new file mode 100644
index 0000000000..a4ce4235ed
--- /dev/null
+++ b/src/common/kmgr_utils.c
@@ -0,0 +1,354 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.c
+ * Shared frontend/backend for cryptographic key management
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/kmgr_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#ifdef FRONTEND
+#include "common/logging.h"
+#endif
+#include "common/kmgr_utils.h"
+#include "common/sha2.h"
+#include "utils/elog.h"
+#include "storage/fd.h"
+
+#define KMGR_PROMPT_MSG "Enter database encryption pass phrase:"
+
+static pg_cipher_ctx *wrapctx = NULL;
+static pg_cipher_ctx *unwrapctx = NULL;
+static bool keywrap_initialized = false;
+
+#ifdef FRONTEND
+static FILE *open_pipe_stream(const char *command);
+static int close_pipe_stream(FILE *file);
+#endif
+
+static void
+initialize_keywrap_ctx(void)
+{
+ wrapctx = pg_cipher_ctx_create();
+ if (wrapctx == NULL)
+ goto err;
+
+ unwrapctx = pg_cipher_ctx_create();
+ if (unwrapctx == NULL)
+ goto err;
+
+ if (!pg_aes256_ctr_wrap_init(wrapctx))
+ goto err;
+
+ if (!pg_aes256_ctr_wrap_init(unwrapctx))
+ goto err;
+
+ keywrap_initialized = true;
+ return;
+
+err:
+#ifdef FRONTEND
+ pg_log_fatal("could not initialize cipher context");
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("could not initialize cipher context")));
+#endif
+}
+
+/*
+ * Hash the given passphrase and extract it into KEK and HMAC
+ * key.
+ */
+void
+kmgr_derive_keys(char *passphrase, Size passlen,
+ uint8 kek[KMGR_KEK_LEN],
+ uint8 hmackey[KMGR_HMAC_KEY_LEN])
+{
+ uint8 keys[PG_SHA512_DIGEST_LENGTH];
+ pg_sha512_ctx ctx;
+
+ pg_sha512_init(&ctx);
+ pg_sha512_update(&ctx, (const uint8 *) passphrase, passlen);
+ pg_sha512_final(&ctx, keys);
+
+ /*
+ * SHA-512 results 64 bytes. We extract it into two keys for
+ * each 32 bytes.
+ */
+ if (kek)
+ memcpy(kek, keys, KMGR_KEK_LEN);
+ if (hmackey)
+ memcpy(hmackey, keys + KMGR_KEK_LEN, KMGR_HMAC_KEY_LEN);
+}
+
+/*
+ * Verify the correctness of the given passphrase. We compute HMACs of the
+ * wrapped key using the HMAC key retrieved from the user provided passphrase.
+ * And then we compare it with the HMAC stored alongside the controlfile. Return
+ * true if both HMACs are matched, meaning the given passphrase is correct.
+ * Otherwise return false.
+ */
+bool
+kmgr_verify_passphrase(char *passphrase, int passlen,
+ WrappedEncKeyWithHmac *kh, int keylen)
+{
+ uint8 user_kek[KMGR_KEK_LEN];
+ uint8 user_hmackey[KMGR_HMAC_KEY_LEN];
+ uint8 result_hmac[KMGR_HMAC_LEN];
+
+ kmgr_derive_keys(passphrase, passlen, user_kek, user_hmackey);
+
+ /* Verify both HMAC */
+ kmgr_compute_HMAC(user_hmackey, kh->key, keylen, result_hmac);
+
+ if (memcmp(result_hmac, kh->hmac, KMGR_HMAC_LEN) != 0)
+ return false;
+
+ return true;
+}
+
+/*
+ * Run cluster passphrase command.
+ *
+ * prompt will be substituted for %p.
+ *
+ * The result will be put in buffer buf, which is of size size.
+ * The return value is the length of the actual result.
+ */
+int
+kmgr_run_cluster_passphrase_command(char *passphrase_command, char *buf,
+ int size)
+{
+ char command[MAXPGPATH];
+ char *p;
+ char *dp;
+ char *endp;
+ FILE *fh;
+ int pclose_rc;
+ size_t len = 0;
+
+ Assert(size > 0);
+ buf[0] = '\0';
+
+ dp = command;
+ endp = command + MAXPGPATH - 1;
+ *endp = '\0';
+
+ for (p = passphrase_command; *p; p++)
+ {
+ if (p[0] == '%')
+ {
+ switch (p[1])
+ {
+ case 'p':
+ StrNCpy(dp, KMGR_PROMPT_MSG, strlen(KMGR_PROMPT_MSG));
+ dp += strlen(KMGR_PROMPT_MSG);
+ p++;
+ break;
+ case '%':
+ p++;
+ if (dp < endp)
+ *dp++ = *p;
+ break;
+ default:
+ if (dp < endp)
+ *dp++ = *p;
+ break;
+ }
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *p;
+ }
+ }
+ *dp = '\0';
+
+#ifdef FRONTEND
+ fh = open_pipe_stream(command);
+ if (fh == NULL)
+ {
+ pg_log_fatal("could not execute command \"%s\": %m",
+ command);
+ exit(EXIT_FAILURE);
+ }
+#else
+ fh = OpenPipeStream(command, "r");
+ if (fh == NULL)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not execute command \"%s\": %m",
+ command)));
+#endif
+
+ if ((len = fread(buf, sizeof(char), size, fh)) < size)
+ {
+ if (ferror(fh))
+ {
+#ifdef FRONTEND
+ pg_log_fatal("could not read from command \"%s\": %m",
+ command);
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not read from command \"%s\": %m",
+ command)));
+#endif
+ }
+ }
+
+#ifdef FRONTEND
+ pclose_rc = close_pipe_stream(fh);
+#else
+ pclose_rc = ClosePipeStream(fh);
+#endif
+
+ if (pclose_rc == -1)
+ {
+#ifdef FRONTEND
+ pg_log_fatal("could not close pipe to external command: %m");
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not close pipe to external command: %m")));
+#endif
+ }
+ else if (pclose_rc != 0)
+ {
+#ifdef FRONTEND
+ pg_log_fatal("command \"%s\" failed", command);
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("command \"%s\" failed",
+ command),
+ errdetail_internal("%s", wait_result_to_str(pclose_rc))));
+#endif
+ }
+
+ return len;
+}
+
+bool
+kmgr_wrap_key(uint8 *key, const uint8 *in, int insize, uint8 *out)
+{
+ int outsize;
+
+ if (!keywrap_initialized)
+ initialize_keywrap_ctx();
+
+ return pg_cipher_encrypt(wrapctx, key, in , insize,
+ NULL, out, &outsize);
+}
+
+bool
+kmgr_unwrap_key(uint8 *key, const uint8 *in, int insize, uint8 *out)
+{
+ int outsize;
+
+ if (!keywrap_initialized)
+ initialize_keywrap_ctx();
+
+ return pg_cipher_decrypt(unwrapctx, key, in, insize,
+ NULL, out, &outsize);
+}
+
+bool
+kmgr_compute_HMAC(uint8 *key, const uint8 *data, int size,
+ uint8 *result)
+{
+ int resultsize;
+
+ return pg_compute_HMAC(key, data, size, result, &resultsize);
+}
+
+/* Convert cipher name string to integer value */
+int
+kmgr_cipher_value(const char *name)
+{
+ if (strcasecmp(name, "aes-128") == 0)
+ return KMGR_ENCRYPTION_AES128;
+
+ if (strcasecmp(name, "aes-256") == 0)
+ return KMGR_ENCRYPTION_AES256;
+
+ return KMGR_ENCRYPTION_OFF;
+}
+
+/* Convert integer value to cipher name string */
+char *
+kmgr_cipher_string(int value)
+{
+ switch (value)
+ {
+ case KMGR_ENCRYPTION_OFF :
+ return "off";
+ case KMGR_ENCRYPTION_AES128:
+ return "aes-128";
+ case KMGR_ENCRYPTION_AES256:
+ return "aes-256";
+ default:
+ break;
+ }
+
+ return "unknown";
+}
+
+#ifdef FRONTEND
+static FILE *
+open_pipe_stream(const char *command)
+{
+ FILE *res;
+
+#ifdef WIN32
+ size_t cmdlen = strlen(command);
+ char *buf;
+ int save_errno;
+
+ buf = malloc(cmdlen + 2 + 1);
+ if (buf == NULL)
+ {
+ errno = ENOMEM;
+ return NULL;
+ }
+ buf[0] = '"';
+ mempcy(&buf[1], command, cmdlen);
+ buf[cmdlen + 1] = '"';
+ buf[cmdlen + 2] = '\0';
+
+ res = _popen(buf, "r");
+
+ save_errno = errno;
+ free(buf);
+ errno = save_errno;
+#else
+ res = popen(command, "r");
+#endif /* WIN32 */
+ return res;
+}
+
+static int
+close_pipe_stream(FILE *file)
+{
+#ifdef WIN32
+ return _pclose(file);
+#else
+ return pclose(file);
+#endif /* WIN32 */
+}
+#endif /* FRONTEND */
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 9b588c87a5..4d036a0570 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -15,6 +15,7 @@
#include "access/xlogdefs.h"
#include "access/xloginsert.h"
#include "access/xlogreader.h"
+#include "crypto/kmgr.h"
#include "datatype/timestamp.h"
#include "lib/stringinfo.h"
#include "nodes/pg_list.h"
@@ -291,8 +292,10 @@ extern TimestampTz GetCurrentChunkReplayStartTime(void);
extern void UpdateControlFile(void);
extern uint64 GetSystemIdentifier(void);
+extern WrappedEncKeyWithHmac *GetMasterEncryptionKey(void);
extern char *GetMockAuthenticationNonce(void);
extern bool DataChecksumsEnabled(void);
+extern int GetDataEncryptionCipher(void);
extern XLogRecPtr GetFakeLSNForUnloggedRel(void);
extern Size XLOGShmemSize(void);
extern void XLOGShmemInit(void);
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index cf7d4485e9..ffc95b62bb 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -17,6 +17,7 @@
#include "access/transam.h"
#include "access/xlogdefs.h"
+#include "crypto/kmgr.h"
#include "pgtime.h" /* for pg_time_t */
#include "port/pg_crc32c.h"
@@ -219,6 +220,9 @@ typedef struct ControlFileData
/* Are data pages protected by checksums? Zero if no checksum version */
uint32 data_checksum_version;
+ /* Are data pages and WAL encrypted? Zero if encryption is disabled */
+ uint32 data_encryption_cipher;
+
/*
* Random nonce, used in authentication requests that need to proceed
* based on values that are cluster-unique, like a SASL exchange that
@@ -226,6 +230,11 @@ typedef struct ControlFileData
*/
char mock_authentication_nonce[MOCK_AUTH_NONCE_LEN];
+ /*
+ * Key information for data encryption.
+ */
+ WrappedEncKeyWithHmac master_dek;
+
/* CRC of all above ... MUST BE LAST! */
pg_crc32c crc;
} ControlFileData;
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index ac8f64b219..5b8d4e23b5 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10729,4 +10729,17 @@
proname => 'pg_partition_root', prorettype => 'regclass',
proargtypes => 'regclass', prosrc => 'pg_partition_root' },
+# function for key managements
+{ oid => '8200', descr => 'rotate cluter encryption key',
+ proname => 'pg_rotate_encryption_key',
+ provolatile => 'v', prorettype => 'bool',
+ proargtypes => '', prosrc => 'pg_rotate_encryption_key' },
+{ oid => '8201', descr => 'wrap the given secret',
+ proname => 'pg_kmgr_wrap',
+ provolatile => 'v', prorettype => 'bytea',
+ proargtypes => 'bytea', prosrc => 'pg_kmgr_wrap' },
+{ oid => '8202', descr => 'unwrap the given secret',
+ proname => 'pg_kmgr_unwrap',
+ provolatile => 'v', prorettype => 'bytea',
+ proargtypes => 'bytea', prosrc => 'pg_kmgr_unwrap' },
]
diff --git a/src/include/common/cipher.h b/src/include/common/cipher.h
new file mode 100644
index 0000000000..c61cb34b1c
--- /dev/null
+++ b/src/include/common/cipher.h
@@ -0,0 +1,70 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.h
+ * Declarations for cryptographic functions
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/cipher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef CIPHER_H
+#define CIPHER_H
+
+#ifdef USE_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#endif
+
+/* Key lengths for AES */
+#define AES128_KEY_LEN 16
+#define AES256_KEY_LEN 32
+
+/*
+ * The encrypted data is a series of blocks of size ENCRYPTION_BLOCK.
+ * Initialization vector(IV) is the same size of cipher block.
+ */
+#define AES_BLOCK_SIZE 16
+#define AES_IV_SIZE (AES_BLOCK_SIZE)
+
+/*
+ * Key wrapping appends the initial 8 bytes value. Therefore
+ * wrapped key size gets larger than original one.
+ */
+#define AES256_KEY_WRAP_VALUE_LEN 8
+#define AES256_MAX_WRAPPED_KEY_LEN (AES256_KEY_LEN + AES256_KEY_WRAP_VALUE_LEN)
+
+/* Size of HMAC key is the same as the length of hash, we use SHA-256 */
+#define SHA256_HMAC_KEY_LEN 32
+
+/* SHA-256 results 256 bits HMAC */
+#define SHA256_HMAC_LEN 32
+
+
+#ifdef USE_OPENSSL
+typedef EVP_CIPHER_CTX pg_cipher_ctx;
+#else
+typedef void pg_cipher_ctx;
+#endif
+
+extern pg_cipher_ctx *pg_cipher_ctx_create(void);
+extern void pg_cipher_setup(void);
+extern bool pg_aes256_ctr_wrap_init(pg_cipher_ctx *ctx);
+extern bool pg_aes256_ctr_unwrap_init(pg_cipher_ctx *ctx);
+
+extern bool pg_cipher_encrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool pg_cipher_decrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool pg_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size);
+
+#endif /* CIPHER_H */
diff --git a/src/include/common/cipher_openssl.h b/src/include/common/cipher_openssl.h
new file mode 100644
index 0000000000..97d8ae766b
--- /dev/null
+++ b/src/include/common/cipher_openssl.h
@@ -0,0 +1,39 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher_openssl.h
+ * Declarations for helper functions using OpenSSL
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/cipher_openssl.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef CIPHER_OPENSSL_H
+#define CIPHER_OPENSSL_H
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+
+extern pg_cipher_ctx *ossl_cipher_ctx_create(void);
+extern bool ossl_cipher_setup(void);
+extern bool ossl_aes256_ctr_wrap_init(pg_cipher_ctx *ctx);
+extern bool ossl_aes256_ctr_unwrap_init(pg_cipher_ctx *ctx);
+extern bool ossl_cipher_encrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool ossl_cipher_decrypt(pg_cipher_ctx *ctx, const uint8 *key,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool ossl_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size);
+#endif
diff --git a/src/include/common/kmgr_utils.h b/src/include/common/kmgr_utils.h
new file mode 100644
index 0000000000..600f3b1b1b
--- /dev/null
+++ b/src/include/common/kmgr_utils.h
@@ -0,0 +1,64 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.h
+ * Declarations for utility function for cryptographic key management
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/kmgr_utils.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_UTILS_H
+#define KMGR_UTILS_H
+
+#include "common/cipher.h"
+
+/* Key encryption key is always AES-256 key */
+#define KMGR_KEK_LEN AES256_KEY_LEN
+
+/* Master data encryption key supports AES-128 and AES-256 */
+#define KMGR_MAX_DEK_LEN AES256_KEY_LEN
+
+/* HMAC and HMAC key */
+#define KMGR_HMAC_KEY_LEN SHA256_HMAC_KEY_LEN
+#define KMGR_HMAC_LEN SHA256_HMAC_LEN
+
+#define KMGR_MAX_PASSPHRASE_LEN 1024
+#define KMGR_MIN_PASSPHRASE_LEN 8
+
+/* Value of data_encryption_cipher */
+enum
+{
+ KMGR_ENCRYPTION_OFF = 0,
+ KMGR_ENCRYPTION_AES128,
+ KMGR_ENCRYPTION_AES256
+};
+
+/*
+ * Struct for keys that needs to be verified using its HMAC.
+ */
+typedef struct WrappedEncKeyWithHmac
+{
+ uint8 key[AES256_MAX_WRAPPED_KEY_LEN];
+ uint8 hmac[KMGR_HMAC_LEN];
+} WrappedEncKeyWithHmac;
+
+extern void kmgr_derive_keys(char *passphrase, Size passlen,
+ uint8 kek[KMGR_KEK_LEN],
+ uint8 hmackey[KMGR_HMAC_KEY_LEN]);
+extern bool kmgr_verify_passphrase(char *passphrase, int passlen,
+ WrappedEncKeyWithHmac *kh, int keylen);
+extern bool kmgr_wrap_key(uint8 *key, const uint8 *in, int insize,
+ uint8 *out);
+extern bool kmgr_unwrap_key(uint8 *key, const uint8 *in, int insize,
+ uint8 *out);
+extern bool kmgr_compute_HMAC(uint8 *key, const uint8 *data, int size,
+ uint8 *result);
+extern int kmgr_run_cluster_passphrase_command(char *passphrase_command,
+ char *buf, int size);
+extern int kmgr_cipher_value(const char *name);
+extern char * kmgr_cipher_string(int value);
+
+#endif /* KMGR_UTILS_H */
diff --git a/src/include/crypto/kmgr.h b/src/include/crypto/kmgr.h
new file mode 100644
index 0000000000..fc41131f69
--- /dev/null
+++ b/src/include/crypto/kmgr.h
@@ -0,0 +1,44 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.h
+ * Key management module for transparent data encryption
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/crypto/kmgr.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_H
+#define KMGR_H
+
+#include "common/cipher.h"
+#include "common/kmgr_utils.h"
+#include "storage/relfilenode.h"
+#include "storage/bufpage.h"
+
+#define KMGR_MAX_PASSPHRASE_LEN 1024
+#define KMGR_PROMPT_MSG "Enter database encryption pass phrase:"
+
+#define DataEncryptionEnabled() \
+ (data_encryption_cipher > KMGR_ENCRYPTION_OFF)
+
+#define SizeOfWrappedDEK() \
+ (EncryptionKeyLen + AES256_KEY_WRAP_VALUE_LEN)
+
+/* GUC parameter */
+extern PGDLLIMPORT int data_encryption_cipher;
+
+/* Encryption keys (TDEK and WDEK) length */
+extern int EncryptionKeyLen;
+
+/* GUC variable */
+extern char *cluster_passphrase_command;
+
+extern WrappedEncKeyWithHmac *BootStrapKmgr(int bootstrap_data_encryption_cipher);
+extern void InitializeKmgr(void);
+extern const char *KmgrGetMasterEncryptionKey(void);
+extern void assign_data_encryption_cipher(int new_encryption_cipher,
+ void *extra);
+
+#endif /* KMGR_H */
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 050c48b108..f0672e3029 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -415,6 +415,9 @@
/* Define to 1 if you have the `OPENSSL_init_ssl' function. */
#undef HAVE_OPENSSL_INIT_SSL
+/* Define to 1 if you have the `OPENSSL_init_crypto' function. */
+#undef HAVE_OPENSSL_INIT_CRYPTO
+
/* Define to 1 if you have the <ossp/uuid.h> header file. */
#undef HAVE_OSSP_UUID_H
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index fe076d823d..fff3c27995 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -895,6 +895,9 @@ typedef enum
WAIT_EVENT_DATA_FILE_TRUNCATE,
WAIT_EVENT_DATA_FILE_WRITE,
WAIT_EVENT_DSM_FILL_ZERO_WRITE,
+ WAIT_EVENT_KMGR_FILE_READ,
+ WAIT_EVENT_KMGR_FILE_SYNC,
+ WAIT_EVENT_KMGR_FILE_WRITE,
WAIT_EVENT_LOCK_FILE_ADDTODATADIR_READ,
WAIT_EVENT_LOCK_FILE_ADDTODATADIR_SYNC,
WAIT_EVENT_LOCK_FILE_ADDTODATADIR_WRITE,
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index d68976fafa..c1ae129f9d 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -89,6 +89,7 @@ enum config_group
STATS,
STATS_MONITORING,
STATS_COLLECTOR,
+ ENCRYPTION,
AUTOVACUUM,
CLIENT_CONN,
CLIENT_CONN_STATEMENT,
