Dear OpenSSL team! I've found a bug when use TLS/1.2+RSA512+DHE cipher
Imagine that TLS/1.2 Client requests the following two Signature/Hash pairs in its signature_algorithms(13) TLS extension of the ClientHello: (RSA+SHA512) and (RSA+SHA256) Imagine also that RSA512 certificate is selected by the server. Since tls1_process_sigalgs() function selects only 1st algorithm from the received ClientHello then server SSL sctucture's cert field will be filled in with SHA512 hash algorithm and RSA512 private key. This algorithms are incompatible since RSA512 can sign only messages not more than 512bits = 64Bytes in length and PKCS1(SHA512) is 83 Bytes in length. Another words server SSL sctucture's cert field will be filled in with CERT_PKEY entry with incompatible privatekey and digest fields. Another hand there is a possibility to choose SHA256 as hash algorithm that is proposed by the Client. There is also a weak error related to the improper digest algorithm choose described above. When ssl3_send_server_key_exchange() function eventually calls RSA_sign() the last function detects RSA512+SHA512 incompatibility and returns an error. ssl3_send_server_key_exchange() after this error returns without sending anything. As a results the connection hangs: client waits for reply to ClientHello but server never send anything. I think better way is to send a handshake failure or any more appropriate alert. If there are no appropriate alerts in TLS/1.2 the custom OpenSSL alert may be sent. I attach 2 patches: patch1.patch - fix of the weak error - sending Handshake Failure alert patch2.patch - the MAIN fix in this email - check digest/privatekey compatibility other attachements: *-orig.pcap - the TLS/1.2+RSA512+DHE session for original openssl-1.0.2f *-patch1.pcap - the same session when patch1.patch is applied *-patch2.pcap - the same session when patch2.patch is applied RootCA.pem - private key to decrypt session if required The session is created by s_server/s_client with following commands: ./openssl s_server -cert RootCA.pem -key RootCA.pem -cipher DHE-RSA-AES256-SHA256 ./openssl s_client -connect localhost:4433 -cipher DHE-RSA-AES256-SHA256 Thanks, Oleg Strelnikov
diff -ur openssl-1.0.1f-orig/ssl/s3_srvr.c openssl-1.0.1f/ssl/s3_srvr.c
--- openssl-1.0.1f-orig/ssl/s3_srvr.c 2014-01-06 17:47:42.000000000 +0400
+++ openssl-1.0.1f/ssl/s3_srvr.c 2014-02-06 15:05:26.000000000 +0400
@@ -1979,8 +1979,9 @@
if (!EVP_SignFinal(&md_ctx,&(p[2]),
(unsigned int *)&i,pkey))
{
+ al=SSL_AD_HANDSHAKE_FAILURE;
SSLerr(SSL_F_SSL3_SEND_SERVER_KEY_EXCHANGE,ERR_LIB_EVP);
- goto err;
+ goto f_err;
}
s2n(i,p);
n+=i+2;
diff -ur openssl-1.0.1f-orig/ssl/t1_lib.c openssl-1.0.1f/ssl/t1_lib.c
--- openssl-1.0.1f-orig/ssl/t1_lib.c 2014-01-06 17:47:42.000000000 +0400
+++ openssl-1.0.1f/ssl/t1_lib.c 2014-02-06 15:27:21.000000000 +0400
@@ -2468,6 +2468,62 @@
}
}
+/* Check the digest/sign compatibility */
+
+static int tls_is_compatible(const EVP_MD *digest, const EVP_PKEY *privatekey)
+ {
+ if (privatekey)
+ switch(privatekey->type)
+ {
+#ifndef OPENSSL_NO_RSA
+ case EVP_PKEY_RSA:
+ switch(digest->type)
+ {
+#ifndef OPENSSL_NO_MD5
+ case NID_md5:
+#ifdef OPENSSL_FIPS
+ if (FIPS_mode())
+ break;
+#endif
+ break;
+#endif
+#ifndef OPENSSL_NO_SHA
+ case NID_sha1:
+ break;
+#endif
+#ifndef OPENSSL_NO_SHA256
+ case NID_sha224:
+ break;
+
+ case NID_sha256:
+ break;
+#endif
+#ifndef OPENSSL_NO_SHA512
+ case NID_sha384:
+ case NID_sha512:
+ if (BN_num_bits(privatekey->pkey.rsa->n) <= 512)
+ return 0;
+ break;
+#endif
+ default:
+ /* break */ ;
+ }
+ break;
+#endif /* OPENSSL_NO_RSA */
+#ifndef OPENSSL_NO_DSA
+ case EVP_PKEY_DSA:
+ break;
+#endif /* OPENSSL_NO_DSA */
+#ifndef OPENSSL_NO_EC
+ case EVP_PKEY_EC:
+ break;
+#endif /* OPENSSL_NO_EC */
+ default:
+ /* break */ ;
+ }
+ return 1;
+ }
+
/* Set preferred digest for each key type */
int tls1_process_sigalgs(SSL *s, const unsigned char *data, int dsize)
@@ -2515,7 +2571,9 @@
if (c->pkeys[idx].digest == NULL)
{
md = tls12_get_hash(hash_alg);
- if (md)
+ if (md && tls_is_compatible(md, c->pkeys[idx].privatekey) &&
+ ((idx != SSL_PKEY_RSA_SIGN) ||
+ tls_is_compatible(md, c->pkeys[SSL_PKEY_RSA_ENC].privatekey)))
{
c->pkeys[idx].digest = md;
if (idx == SSL_PKEY_RSA_SIGN)
dhe+rsa512+sha512-orig.pcap
Description: application/vnd.tcpdump.pcap
dhe+rsa512+sha512-patch1.pcap
Description: application/vnd.tcpdump.pcap
dhe+rsa512+sha512-patch2.pcap
Description: application/vnd.tcpdump.pcap
RootCA.pem
Description: application/x509-ca-cert
