Bug#828593: virtualbox: FTBFS with openssl 1.1.0
On 2016-09-12 22:10:34 [+0200], Sebastian Andrzej Siewior wrote: > VirtualBox hackers! Could you please take a look at this? It should > compile against openssl 1.1.0 and 1.0.2h and work. It seems the patch is/was not perfect. According to [0] they applied parts of it to trunk and will backport it for their next 5.1.x release. \o/ [0] https://www.virtualbox.org/pipermail/vbox-dev/2016-September/014083.html > > Kurt Sebastian
Bug#828593: virtualbox: FTBFS with openssl 1.1.0
On 2016-09-10 01:30:13 [+0200], Kurt Roeckx wrote:
> The code seems to be doing: The signature algorithm was
> sha1WithRSAEncryption, so the public key they passed us must
> have been an RSA key. But from what I understand, what the code
> really did was: it's an EVP_sha1(), it's an an RSA key. But if it
> was EVP_sha256(), it was an unknown key type. What they do might
> have worked in the past by accident. But it clearly doesn't work
> properly.
>
> I don't think there is a way of doing what they want to do. They
> should really just pass the type of the public key to the
> function.
I think I've found it a few lines above.
VirtualBox hackers! Could you please take a look at this? It should
compile against openssl 1.1.0 and 1.0.2h and work.
> Kurt
Sebastian
>From 0e3d6cc69959fdf4f4fd56f65353b76897c0daad Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior
Date: Wed, 7 Sep 2016 20:22:54 +
Subject: [PATCH] virtualbox 5.1.4: get it built against openssl-1.1
Signed-off-by: Sebastian Andrzej Siewior
---
src/VBox/RDP/client-1.8.3/ssl.c| 70 +-
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++--
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x1010
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len
Bug#828593: virtualbox: FTBFS with openssl 1.1.0
On Wed, Sep 07, 2016 at 11:38:01PM +0200, Sebastian Andrzej Siewior wrote: > On 2016-06-26 12:24:40 [+0200], Kurt Roeckx wrote: > > If you have problems making things work, feel free to contact us. > > HALP! > > It builds against old & new ssl and I am proud what I managed in > rdssl_cert_to_rkey(). However they dereference > EVP_MD->required_pkey_type which vanished in 1.1 and I have no idea what > it was and its purpose was. They don't even assign anything to it in > 1.0.2h. I think this is the relevant commit: commit 7f572e958b13041056f377a62d3219633cfb1e8a Author: Dr. Stephen Henson Date: Wed Dec 2 13:57:04 2015 + Remove legacy sign/verify from EVP_MD. Remove sign/verify and required_pkey_type fields of EVP_MD: these are a legacy from when digests were linked to public key types. All signing is now handled by the corresponding EVP_PKEY_METHOD. Only allow supported digest types in RSA EVP_PKEY_METHOD: other algorithms already block unsupported types. Remove now obsolete EVP_dss1() and EVP_ecdsa(). Reviewed-by: Richard Levitte The code seems to be doing: The signature algorithm was sha1WithRSAEncryption, so the public key they passed us must have been an RSA key. But from what I understand, what the code really did was: it's an EVP_sha1(), it's an an RSA key. But if it was EVP_sha256(), it was an unknown key type. What they do might have worked in the past by accident. But it clearly doesn't work properly. I don't think there is a way of doing what they want to do. They should really just pass the type of the public key to the function. Kurt
Bug#828593: virtualbox: FTBFS with openssl 1.1.0
On 2016-06-26 12:24:40 [+0200], Kurt Roeckx wrote:
> If you have problems making things work, feel free to contact us.
HALP!
It builds against old & new ssl and I am proud what I managed in
rdssl_cert_to_rkey(). However they dereference
EVP_MD->required_pkey_type which vanished in 1.1 and I have no idea what
it was and its purpose was. They don't even assign anything to it in
1.0.2h.
> Kurt
Sebastian
>From 0e3d6cc69959fdf4f4fd56f65353b76897c0daad Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior
Date: Wed, 7 Sep 2016 20:22:54 +
Subject: [PATCH] virtualbox 5.1.4: get it built against openssl-1.1
Signed-off-by: Sebastian Andrzej Siewior
---
src/VBox/RDP/client-1.8.3/ssl.c| 70 +-
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++--
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x1010
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
- HMAC_CTX_cleanup(&ctx);
}
diff --git a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
index ddfccec..fa19fa7 100644
--- a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
+++ b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
@@ -76,26 +76,28 @@ PFNRT g_VBoxRTDeps[] =
(PFNRT)i2d_X509,
(PFNRT)i2d_X509,
(PFNRT)i2d_Publ
Bug#828593: virtualbox: FTBFS with openssl 1.1.0
Source: virtualbox Version: 5.0.20-dfsg-2 Severity: important Control: block 827061 by -1 Hi, OpenSSL 1.1.0 is about to released. During a rebuild of all packages using OpenSSL this package fail to build. A log of that build can be found at: https://breakpoint.cc/openssl-1.1-rebuild-2016-05-29/Attempted/virtualbox_5.0.20-dfsg-2_amd64-20160530-2117 On https://wiki.openssl.org/index.php/1.1_API_Changes you can see various of the reasons why it might fail. There are also updated man pages at https://www.openssl.org/docs/manmaster/ that should contain useful information. There is a libssl-dev package available in experimental that contains a recent snapshot, I suggest you try building against that to see if everything works. If you have problems making things work, feel free to contact us. Kurt
