A colleague of mine has experienced the following problem with a custom SSL verification hook we developed for Apache 2.0. Does any of you guys have an idea about why the Apache child processes start using 100% CPU?
We have implemented a custom SSL certificate verification mechanism with an optional hook. If the SSLVerifyClient directive is set to "custom", we set our callback instead of the default ssl_callback_SSLVerify (patch follows). If we use IE as client, the httpd process starts to race, with this backtrace:
[Switching to Thread 16384 (LWP 12156)]
0x08083eab in ssl_io_input_read (inctx=0x822f320, buf=0x822f348 "", len=0xbffff0e8)
at ssl_engine_io.c:621
621 if (!inctx->filter_ctx->pssl) {
(gdb) backtrace
#0 0x08083eab in ssl_io_input_read (inctx=0x822f320, buf=0x822f348 "", len=0xbffff0e8)
at ssl_engine_io.c:621
#1 0x080840a3 in ssl_io_input_getline (inctx=0x822f320, buf=0x822f348 "",
len=0xbffff118) at ssl_engine_io.c:744
#2 0x08084926 in ssl_io_filter_input (f=0x8231350, bb=0x8233018, mode=AP_MODE_GETLINE,
block=APR_BLOCK_READ, readbytes=0) at ssl_engine_io.c:1253
#3 0x080a3fc6 in ap_get_brigade (next=0x8231350, bb=0x8233018, mode=AP_MODE_GETLINE,
block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514
#4 0x080a3fc6 in ap_get_brigade (next=0x8232fb8, bb=0x8233018, mode=AP_MODE_GETLINE,
block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514
#5 0x080a4dc3 in ap_rgetline_core (s=0x82323c0, n=8192, read=0xbffff23c, r=0x82323a8,
fold=0, bb=0x8233018) at protocol.c:256
#6 0x080a5285 in read_request_line (r=0x82323a8, bb=0x8233018) at protocol.c:62
3
#7 0x080a57f3 in ap_read_request (conn=0x8220460) at protocol.c:885
#8 0x080914a4 in ap_process_http_connection (c=0x8220460) at http_core.c:312
#9 0x080a21de in ap_run_process_connection (c=0x8220460) at connection.c:85
#10 0x08097f95 in child_main (child_num_arg=2) at prefork.c:696
#11 0x080980dc in make_child (s=0x8194378, slot=2) at prefork.c:790
#12 0x08098132 in startup_children (number_to_start=30) at prefork.c:808
#13 0x08098434 in ap_mpm_run (_pconf=0x81910f0, plog=0x81cd1e0, s=0x8194378)
at prefork.c:1024
#14 0x0809d6db in main (argc=5, argv=0xbffff484) at main.c:665
#15 0x4014f7e6 in __libc_start_main () from /lib/libc.so.6
(gdb) detachIf we use the standard SSLVerifyClient (require), this does not happen. With ssldump we obtained following conversation:
with SSLVerifyClient custom:
knudde engine # ssldump -N -d -k
/home/jos/tmp/henhouse/notary-certs/notary_server_nopp.key port 443
New TCP connection #1: petillant.be.ubizen.com(2419) <-> knudde(443)
1 1 0.0206 (0.0206) C>S SSLv2 compatible client hello
Version 3.0
cipher suites
SSL_RSA_WITH_RC4_128_MD5
SSL_RSA_WITH_RC4_128_SHA
SSL_RSA_WITH_3DES_EDE_CBC_SHA
SSL2_CK_RC4
SSL2_CK_3DES
SSL2_CK_RC2
SSL_RSA_WITH_DES_CBC_SHA
SSL2_CK_DES
SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
SSL_RSA_EXPORT_WITH_RC4_40_MD5
SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
SSL2_CK_RC4_EXPORT40
SSL2_CK_RC2_EXPORT40
SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
SSL_DHE_DSS_WITH_DES_CBC_SHA
SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
1 2 0.1176 (0.0969) S>C Handshake
ServerHello
Version 3.0
session_id[32]=
44 2b 61 11 2f bb 2a 48 2f 33 70 3b d8 8e 22 1f
e2 b4 1a ec 88 1f f0 ce ce 6d 8d f5 6f d6 8a 0f
cipherSuite SSL_RSA_WITH_RC4_128_MD5
compressionMethod NULL
1 3 0.1176 (0.0000) S>C Handshake
Certificate
Subject
C=BE
C=BE
ST=Some-State
L=Leive
O=Ubizen
OU=AppSec
CN=notary Server cert
Issuer
C=BE
ST=Some-State
O=Ubizen
OU=AppSec
CN=notary CA
Serial 01
1 4 0.1176 (0.0000) S>C Handshake
CertificateRequest
certificate_types rsa_sign
certificate_types dss_sign
ServerHelloDone
1 0.5964 (0.4788) C>S TCP FIN
1 0.5991 (0.0026) S>C TCP FIN
New TCP connection #2: petillant.be.ubizen.com(2423) <-> knudde(443)
2 1 0.0011 (0.0011) C>S SSLv2 compatible client hello
Version 3.0
cipher suites
SSL_RSA_WITH_RC4_128_MD5
SSL_RSA_WITH_RC4_128_SHA
SSL_RSA_WITH_3DES_EDE_CBC_SHA
SSL2_CK_RC4
SSL2_CK_3DES
SSL2_CK_RC2
SSL_RSA_WITH_DES_CBC_SHA
SSL2_CK_DES
SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
SSL_RSA_EXPORT_WITH_RC4_40_MD5
SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
SSL2_CK_RC4_EXPORT40
SSL2_CK_RC2_EXPORT40
SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
SSL_DHE_DSS_WITH_DES_CBC_SHA
SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
2 2 0.0555 (0.0543) S>C Handshake
ServerHello
Version 3.0
session_id[32]=
54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
cipherSuite SSL_RSA_WITH_RC4_128_MD5
compressionMethod NULL
2 3 0.0555 (0.0000) S>C Handshake
Certificate
Subject
C=BE
ST=Some-State
L=Leive
O=Ubizen
OU=AppSec
CN=notary Server cert
Issuer
C=BE
ST=Some-State
O=Ubizen
OU=AppSec
CN=notary CA
Serial 01
2 4 0.0555 (0.0000) S>C Handshake
CertificateRequest
certificate_types rsa_sign
certificate_types dss_sign
ServerHelloDone
2 5 0.0583 (0.0028) C>S Alert
level warning
value unknown value
2 6 0.0583 (0.0000) C>S Handshake
ClientKeyExchange
2 7 0.0583 (0.0000) C>S ChangeCipherSpec
2 8 0.0583 (0.0000) C>S Handshake
Finished
2 9 0.1808 (0.1224) S>C ChangeCipherSpec
2 10 0.1808 (0.0000) S>C Handshake
Finished
2 0.4370 (0.2562) C>S TCP FIN
New TCP connection #3: petillant.be.ubizen.com(2424) <-> knudde(443)
3 1 0.0009 (0.0009) C>S Handshake
ClientHello
Version 3.0
resume [32]=
54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
cipher suites
SSL_RSA_WITH_RC4_128_MD5
SSL_RSA_WITH_RC4_128_SHA
SSL_RSA_WITH_3DES_EDE_CBC_SHA
SSL_DHE_DSS_WITH_DES_CBC_SHA
SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
compression methods
NULL
3 2 0.0026 (0.0016) S>C Handshake
ServerHello
Version 3.0
session_id[32]=
54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
cipherSuite SSL_RSA_WITH_RC4_128_MD5
compressionMethod NULL
3 3 0.0026 (0.0000) S>C ChangeCipherSpec
3 4 0.0026 (0.0000) S>C Handshake
Finished
3 5 0.0037 (0.0011) C>S ChangeCipherSpec
3 6 0.0037 (0.0000) C>S Handshake
Finished
3 7 0.0092 (0.0054) C>S application_data
---------------------------------------------------------------
GET / HTTP/1.1
Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd
.ms-powerpoint, application/vnd.ms-excel, application/msword, */*
Accept-Language: en-us
Accept-Encoding: gzip, deflate
User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)
Host: knudde.be.ubizen.com
Connection: Keep-Alive---------------------------------------------------------------
and then the apache starts using 100% CPU (well, actually 2 apache children are racing).
with SSLVerifyClient require:
knudde engine # ssldump -N -d -k
/home/jos/tmp/henhouse/notary-certs/notary_server_nopp.key port 443
New TCP connection #1: petillant.be.ubizen.com(2444) <-> knudde(443)
1 1 0.0010 (0.0010) C>S Handshake
ClientHello
Version 3.0
resume [32]=
96 6c 33 38 63 7a 05 f7 38 02 a6 e6 02 84 06 bf
f1 e0 a1 81 41 8c 23 c4 83 e5 63 37 6e 16 61 32
cipher suites
SSL_RSA_WITH_RC4_128_MD5
SSL_RSA_WITH_RC4_128_SHA
SSL_RSA_WITH_3DES_EDE_CBC_SHA
SSL_RSA_WITH_DES_CBC_SHA
SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
SSL_RSA_EXPORT_WITH_RC4_40_MD5
SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
SSL_DHE_DSS_WITH_DES_CBC_SHA
SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
compression methods
NULL
1 2 0.0050 (0.0039) S>C Handshake
ServerHello
Version 3.0
session_id[32]=
0e ab 2d f6 f6 1e 55 21 3f 20 83 a9 30 0c b6 d1
14 74 25 24 2f 66 0d 9c a6 61 63 85 73 86 68 7a
cipherSuite SSL_RSA_WITH_RC4_128_MD5
compressionMethod NULL
1 3 0.0050 (0.0000) S>C Handshake
Certificate
Subject
C=BE
ST=Some-State
O=Ubizen
OU=AppSec
CN=notary CA
Serial 00
Extensions
Extension: X509v3 Subject Key Identifier
Extension: X509v3 Authority Key Identifier
Extension: X509v3 Basic Constraints
Extension: X509v3 Key Usage
Extension: Netscape Cert Type
1 4 0.0050 (0.0000) S>C Handshake
CertificateRequest
certificate_types rsa_sign
certificate_types dss_sign
certificate_authority
C=BE
ST=Some-State
O=Ubizen
OU=AppSec
CN=notary CA
ServerHelloDone
1 5 0.0085 (0.0034) C>S Alert
level warning
value unknown value
1 6 0.0085 (0.0000) C>S Handshake
ClientKeyExchange
1 7 0.0085 (0.0000) C>S ChangeCipherSpec
1 8 0.0085 (0.0000) C>S Handshake
Finished
1 9 0.0312 (0.0227) S>C Alert
level fatal
value handshake_failure
1 0.0318 (0.0005) S>C TCP FIN
1 0.0318 (0.0000) C>S TCP FINWe think that since IE closes the connection (since cancel was clicked on the choose certificate dialog), this messes up the state inside mod_ssl or OpenSSL, which is going to shutdown the connection because of no certificate. After that IE tries again to resume the session, and send the HTTP request.
Relevant changes:
diff -ru httpd-2.0.46/modules/ssl/mod_ssl.h httpd-2.0.46-PATCHED/modules/ssl/mod _ssl.h --- httpd-2.0.46/modules/ssl/mod_ssl.h 2003-05-16 20:12:18.000000000 +0200 +++ httpd-2.0.46-PATCHED/modules/ssl/mod_ssl.h 2003-08-22 10:59:23.000000000 +0 200 @@ -306,7 +306,8 @@ SSL_CVERIFY_NONE = 0, SSL_CVERIFY_OPTIONAL = 1, SSL_CVERIFY_REQUIRE = 2, - SSL_CVERIFY_OPTIONAL_NO_CA = 3 + SSL_CVERIFY_OPTIONAL_NO_CA = 3, + SSL_CVERIFY_CUSTOM = 4 } ssl_verify_t;
#define SSL_VERIFY_PEER_STRICT \
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_config.c httpd-2.0.46-PATCHED/modul
es/ssl/ssl_engine_config.c
--- httpd-2.0.46/modules/ssl/ssl_engine_config.c 2003-03-31 01:17:22.0000
00000 +0200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_config.c 2003-08-22 10:59
:23.000000000 +0200
@@ -912,6 +920,9 @@
else if (strcEQ(arg, "optional_no_ca")) {
*id = SSL_CVERIFY_OPTIONAL_NO_CA;
}
+ else if (strEQ(arg, "custom")) {
+ *id = SSL_CVERIFY_CUSTOM;
+ }
else {
return apr_pstrcat(parms->temp_pool, parms->cmd->name,
": Invalid argument '", arg, "'",
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_init.c httpd-2.0.46-PATCHED/modules
/ssl/ssl_engine_init.c
--- httpd-2.0.46/modules/ssl/ssl_engine_init.c 2003-05-16 20:12:18.000000000 +0
200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_init.c 2003-08-22 10:59:23.0000
00000 +0200
@@ -60,6 +60,7 @@
see Recursive.''
-- Unknown */
#include "mod_ssl.h"
+APR_DECLARE_OPTIONAL_FN(int, custom_ssl_verify, (int ok, X509_STORE_CTX *ctx)); /* _________________________________________________________________
**
@@ -539,12 +543,23 @@
} if ((mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL) ||
- (mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL_NO_CA))
+ (mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL_NO_CA) ||
+ (mctx->auth.verify_mode == SSL_CVERIFY_CUSTOM))
{
verify |= SSL_VERIFY_PEER;
}+ {
+ APR_OPTIONAL_FN_TYPE(custom_ssl_verify) *cust_verify = NULL;
+ cust_verify = APR_RETRIEVE_OPTIONAL_FN(custom_ssl_verify);
+ if (mctx->auth.verify_mode == SSL_CVERIFY_CUSTOM && cust_verify) {
+ //verify |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+ verify |= SSL_VERIFY_CLIENT_ONCE;
+ SSL_CTX_set_verify(ctx, verify, cust_verify);
+ } else {
SSL_CTX_set_verify(ctx, verify, ssl_callback_SSLVerify);
+ }
+ } /*
* Configure Client Authentication details
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_io.c httpd-2.0.46-PATCHED/modules/s
sl/ssl_engine_io.c
--- httpd-2.0.46/modules/ssl/ssl_engine_io.c 2003-04-05 21:04:43.000000000 +0
200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_io.c 2003-08-22 11:38:46.0000
00000 +0200
@@ -1176,7 +1176,8 @@
* Make really sure that when a peer certificate
* is required we really got one... (be paranoid)
*/
- if ((sc->server->auth.verify_mode == SSL_CVERIFY_REQUIRE) &&
+ if ((sc->server->auth.verify_mode == SSL_CVERIFY_REQUIRE ||
+ sc->server->auth.verify_mode == SSL_CVERIFY_CUSTOM) &&
!sslconn->client_cert)
{
ap_log_error(APLOG_MARK, APLOG_INFO, 0, c->base_server,
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_kernel.c httpd-2.0.46-PATCHED/modul
es/ssl/ssl_engine_kernel.c
--- httpd-2.0.46/modules/ssl/ssl_engine_kernel.c 2003-05-16 20:12:18.0000
00000 +0200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_kernel.c 2003-08-22 10:59
:23.000000000 +0200
@@ -63,6 +63,8 @@
-- Unknown */
#include "mod_ssl.h"+APR_DECLARE_OPTIONAL_FN(int, custom_ssl_verify, (int ok, X509_STORE_CTX *ctx));
+
/*
* Post Read Request Handler
*/
@@ -423,12 +425,23 @@
} if ((dc->nVerifyClient == SSL_CVERIFY_OPTIONAL) ||
- (dc->nVerifyClient == SSL_CVERIFY_OPTIONAL_NO_CA))
+ (dc->nVerifyClient == SSL_CVERIFY_OPTIONAL_NO_CA) ||
+ (dc->nVerifyClient == SSL_CVERIFY_CUSTOM))
{
verify |= SSL_VERIFY_PEER;
}+ {
+ APR_OPTIONAL_FN_TYPE(custom_ssl_verify) *cust_verify = NULL;
+ cust_verify = APR_RETRIEVE_OPTIONAL_FN(custom_ssl_verify);
+ if (dc->nVerifyClient == SSL_CVERIFY_CUSTOM && cust_verify) {
+ //verify |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+ verify |= SSL_VERIFY_CLIENT_ONCE;
+ modssl_set_verify(ssl, verify, cust_verify);
+ } else {
modssl_set_verify(ssl, verify, ssl_callback_SSLVerify);
+ }
+ }
SSL_set_verify_result(ssl, X509_V_OK); /* determine whether we've to force a renegotiation */
@@ -729,7 +742,8 @@
* Finally check for acceptable renegotiation results
*/
if (dc->nVerifyClient != SSL_CVERIFY_NONE) {
- BOOL do_verify = (dc->nVerifyClient == SSL_CVERIFY_REQUIRE);
+ BOOL do_verify = (dc->nVerifyClient == SSL_CVERIFY_REQUIRE ||
+ dc->nVerifyClient == SSL_CVERIFY_CUSTOM); if (do_verify && (SSL_get_verify_result(ssl) != X509_V_OK)) {
ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,-- ir. Kris Verbeeck Software Engineer
Ubizen - Ubicenter - Philipssite 5 - 3001 Leuven - Belgium T: +32 16 28 70 64 F: +32 16 28 70 77
Ubizen - We Secure e-business - www.ubizen.com
