On 2016-09-10 01:30:13 [+0200], Kurt Roeckx wrote:
> The code seems to be doing: The signature algorithm was
> sha1WithRSAEncryption, so the public key they passed us must
> have been an RSA key. But from what I understand, what the code
> really did was: it's an EVP_sha1(), it's an an RSA key. But if it
> was EVP_sha256(), it was an unknown key type. What they do might
> have worked in the past by accident. But it clearly doesn't work
> properly.
>
> I don't think there is a way of doing what they want to do. They
> should really just pass the type of the public key to the
> function.
I think I've found it a few lines above.
VirtualBox hackers! Could you please take a look at this? It should
compile against openssl 1.1.0 and 1.0.2h and work.
> Kurt
Sebastian
>From 0e3d6cc69959fdf4f4fd56f65353b76897c0daad Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
Date: Wed, 7 Sep 2016 20:22:54 +0000
Subject: [PATCH] virtualbox 5.1.4: get it built against openssl-1.1
Signed-off-by: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
---
src/VBox/RDP/client-1.8.3/ssl.c | 70 +++++++++++++++++---------
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++++++++++------
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x10100000
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+ const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
- HMAC_CTX_cleanup(&ctx);
}
diff --git a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
index ddfccec..fa19fa7 100644
--- a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
+++ b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
@@ -76,26 +76,28 @@ PFNRT g_VBoxRTDeps[] =
(PFNRT)i2d_X509,
(PFNRT)i2d_X509,
(PFNRT)i2d_PublicKey,
+#if OPENSSL_VERSION_NUMBER < 0x10100000
(PFNRT)RSA_generate_key,
- (PFNRT)RSA_generate_key_ex,
(PFNRT)DH_generate_parameters,
- (PFNRT)DH_generate_parameters_ex,
- (PFNRT)RAND_load_file,
(PFNRT)CRYPTO_set_dynlock_create_callback,
(PFNRT)CRYPTO_set_dynlock_lock_callback,
(PFNRT)CRYPTO_set_dynlock_destroy_callback,
+ (PFNRT)SSL_library_init,
+ (PFNRT)SSL_load_error_strings,
+ (PFNRT)TLSv1_server_method,
+#endif
+ (PFNRT)RSA_generate_key_ex,
+ (PFNRT)DH_generate_parameters_ex,
+ (PFNRT)RAND_load_file,
(PFNRT)RTAssertShouldPanic,
(PFNRT)ASMAtomicReadU64,
(PFNRT)ASMAtomicCmpXchgU64,
(PFNRT)ASMBitFirstSet,
(PFNRT)RTBldCfgRevision,
(PFNRT)SSL_free,
- (PFNRT)SSL_library_init,
- (PFNRT)SSL_load_error_strings,
(PFNRT)SSL_CTX_free,
(PFNRT)SSL_CTX_use_certificate_file,
(PFNRT)SSLv23_method,
- (PFNRT)TLSv1_server_method,
NULL
};
diff --git a/src/VBox/Runtime/common/crypto/pkix-verify.cpp b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
index ba537d5..924005b 100644
--- a/src/VBox/Runtime/common/crypto/pkix-verify.cpp
+++ b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
@@ -124,28 +124,39 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
"EVP_get_digestbyname failed on %s (%s)", pszAlogSn, pAlgorithm->szObjId);
/* Initialize the EVP message digest context. */
- EVP_MD_CTX EvpMdCtx;
- EVP_MD_CTX_init(&EvpMdCtx);
- if (!EVP_VerifyInit_ex(&EvpMdCtx, pEvpMdType, NULL /*engine*/))
+ EVP_MD_CTX *EvpMdCtx;
+
+ EvpMdCtx = EVP_MD_CTX_create();
+
+ if (!EvpMdCtx)
+ return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
+ "EVP_MD_CTX_create failed");
+
+ if (!EVP_VerifyInit_ex(EvpMdCtx, pEvpMdType, NULL /*engine*/)) {
+ EVP_MD_CTX_destroy(EvpMdCtx);
return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
"EVP_VerifyInit_ex failed (algorithm type is %s / %s)", pszAlogSn, pAlgorithm->szObjId);
+ }
/* Create an EVP public key. */
int rcOssl;
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Digest the data. */
- EVP_VerifyUpdate(&EvpMdCtx, pvData, cbData);
+ EVP_VerifyUpdate(EvpMdCtx, pvData, cbData);
/* Verify the signature. */
- if (EVP_VerifyFinal(&EvpMdCtx,
+ if (EVP_VerifyFinal(EvpMdCtx,
RTASN1BITSTRING_GET_BIT0_PTR(pSignatureValue),
RTASN1BITSTRING_GET_BYTE_SIZE(pSignatureValue),
pEvpPublicKey) > 0)
@@ -158,13 +169,13 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
- EVP_MD_CTX_cleanup(&EvpMdCtx);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", iAlgoNid);
+ EVP_MD_CTX_destroy(EvpMdCtx);
/*
* Check the result.
@@ -261,11 +272,14 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Create an EVP public key context we can use to validate the digest. */
EVP_PKEY_CTX *pEvpPKeyCtx = EVP_PKEY_CTX_new(pEvpPublicKey, NULL);
@@ -306,12 +320,12 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", iAlgoNid);
/*
* Check the result.
--
2.9.3
