Hello,
I'm running an apache-service using OpenSSL on Linux and Solaris (see full
specifications at the bottom).
On initialisation, all certificate and CA-chains are loaded into OpenSSL. Here
the debug-trace:
(gdb) where
#0 sk_insert (st=0x820a1c0, data=0x8204b08 "\001", loc=0) at stack.c:146
#1 0x401413c1 in sk_push (st=0x820a1c0, data=0x8204b08 "\001") at stack.c:259
#2 0x4017d500 in X509_STORE_add_cert (ctx=0x8209fb0, x=0x82021f0) at
x509_lu.c:353
#3 0x4017fd00 in X509_load_cert_crl_file (ctx=0x820a2d0, file=0x81ca098
"cachain.pem", type=1) at by_file.c:321
#4 0x4017f70a in by_file_ctrl (ctx=0x820a2d0, cmd=1, argp=0x81ca098
"cachain.pem", argl=1, ret=0x0) at by_file.c:123
#5 0x4017cdb3 in X509_LOOKUP_ctrl (ctx=0x820a2d0, cmd=1, argc=0x81ca098
"cachain.pem", argl=1, ret=0x0) at x509_lu.c:117
#6 0x40175ade in X509_STORE_load_locations (ctx=0x8209fb0, file=0x81ca098
"cachain.pem", path=0x0) at x509_d2.c:145
#7 0x401ef8fc in SSL_CTX_load_verify_locations (ctx=0x82020f0,
CAfile=0x81ca098 "cachain.pem", CApath=0x0) at ssl_lib.c:2535
#8 0x402ac90b in ssl_init_ctx_verify (s=0x81c9278, p=0x8170f18,
ptemp=0x8174f28, mctx=0x81ca160)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_init.c:517
#9 0x402ace4f in ssl_init_ctx (s=0x81c9278, p=0x8170f18, ptemp=0x8174f28,
mctx=0x81ca160)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_init.c:686
#10 0x402ad96a in ssl_init_server_ctx (s=0x81c9278, p=0x8170f18,
ptemp=0x8174f28, sc=0x81ca0e8)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_init.c:964
#11 0x402ada0e in ssl_init_ConfigureServer (s=0x81c9278, p=0x8170f18,
ptemp=0x8174f28, sc=0x81ca0e8)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_init.c:982
#12 0x402ac14f in ssl_init_Module (p=0x8170f18, plog=0x81ab000,
ptemp=0x8174f28, base_server=0x8178038)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_init.c:275
#13 0x4020c353 in ap_run_post_config (pconf=0x8170f18, plog=0x81ab000,
ptemp=0x8174f28, s=0x8178038)
at ...apache/httpd-2.2.X/server/config.c:94
#14 0x4021ef0d in ap_main (argc=4, argv=0x80bcff0) at
...apache/httpd-2.2.X/server/main.c:722
#15 0x08049795 in main (argc=5, argv=0xbfffed44) at httpd_2_2_x/navajosv.cpp:117
Afterwards apache will wait for requests in several threads. Once a request
comes in, apache will call SSL_Accept(). Here the debug-trace:
(gdb) print st->sorted
$68 = 0
(gdb) where
#0 sk_sort (st=0x820a1c0) at stack.c:327
#1 0x401412f5 in internal_find (st=0x820a1c0, data=0x5f7a4d4c "\001",
ret_val_options=2) at stack.c:233
#2 0x4014137a in sk_find (st=0x820a1c0, data=0x5f7a4d4c "\001") at stack.c:250
#3 0x4017d7ec in X509_OBJECT_idx_by_subject (h=0x820a1c0, type=1,
name=0x82073a0) at x509_lu.c:456
#4 0x4017d824 in X509_OBJECT_retrieve_by_subject (h=0x820a1c0, type=1,
name=0x82073a0) at x509_lu.c:463
#5 0x4017d30d in X509_STORE_get_by_subject (vs=0x5f7a4e04, type=1,
name=0x82073a0, ret=0x5f7a4dfc) at x509_lu.c:289
#6 0x401ddfb0 in ssl3_output_cert_chain (s=0x851ba98, x=0x8207628) at
s3_both.c:323
#7 0x401d43b6 in ssl3_send_server_certificate (s=0x851ba98) at s3_srvr.c:2655
#8 0x401d0663 in ssl3_accept (s=0x851ba98) at s3_srvr.c:329
#9 0x401ec8a1 in SSL_accept (s=0x851ba98) at ssl_lib.c:869
#10 0x401df694 in ssl23_get_client_hello (s=0x851ba98) at s23_srvr.c:590
#11 0x401debda in ssl23_accept (s=0x851ba98) at s23_srvr.c:216
#12 0x401ec8a1 in SSL_accept (s=0x851ba98) at ssl_lib.c:869
#13 0x402afa47 in ssl_io_filter_connect (filter_ctx=0x85180e8) at
...apache/httpd-2.2.X/modules/ssl/ssl_engine_io.c:1069
#14 0x402b027c in ssl_io_filter_input (f=0x8521028, bb=0x8522df0,
mode=AP_MODE_GETLINE, block=APR_BLOCK_READ, readbytes=0)
at ...apache/httpd-2.2.X/modules/ssl/ssl_engine_io.c:1315
#15 0x4022e1ae in ap_get_brigade (next=0x8521028, bb=0x8522df0,
mode=AP_MODE_GETLINE, block=APR_BLOCK_READ, readbytes=0)
at ...apache/httpd-2.2.X/server/util_filter.c:489
#16 0x40220e7b in ap_rgetline_core (s=0x85220a0, n=5122, read=0x5f7a51d8,
r=0x8522088, fold=0, bb=0x8522df0)
at ...apache/httpd-2.2.X/server/protocol.c:231
#17 0x402215ab in read_request_line (r=0x8522088, bb=0x8522df0) at
...apache/httpd-2.2.X/server/protocol.c:596
#18 0x40221f71 in ap_read_request (conn=0x8517c90) at
...apache/httpd-2.2.X/server/protocol.c:891
#19 0x402d3f86 in ap_process_http_connection (c=0x8517c90) at
...apache/httpd-2.2.X/modules/http/http_core.c:183
#20 0x402111ba in ap_run_process_connection (c=0x8517c90) at
...apache/httpd-2.2.X/server/connection.c:43
#21 0x40211645 in ap_process_connection (c=0x8517c90, csd=0x8517ae0) at
...apache/httpd-2.2.X/server/connection.c:178
#22 0x40233236 in process_socket (p=0x8517aa0, sock=0x8517ae0, my_child_num=0,
my_thread_num=399, bucket_alloc=0x8519aa8)
at ...apache/httpd-2.2.X/server/mpm/worker/worker.c:544
#23 0x40233be7 in worker_thread (thd=0x849dc68, dummy=0x8214380) at
...apache/httpd-2.2.X/server/mpm/worker/worker.c:894
#24 0x4138df88 in dummy_worker (opaque=0x849dc68) at
...apache/httpd-2.2.X/srclib/apr/threadproc/unix/thread.c:142
#25 0x400541b5 in start_thread () from /lib/libpthread.so.0
#26 0x412e83be in clone () from /lib/libc.so.6
As you can see, the cert-chain will be sorted on the very first request,
overwriting the stack-data (without locking).
If we have two (or more) requests at the same time for the very first time,
each thread will call qsort() which leads to undefined behaviour.
I would apply the following fix:
file openssl/crypto/x509/x509_lu.c:
function X509_STORE_add_cert(X509_STORE *ctx, X509 *x):
int X509_STORE_add_cert(X509_STORE *ctx, X509 *x)
{
...
if (X509_OBJECT_retrieve_match(ctx->objs, obj))
{
...
}
else {
sk_X509_OBJECT_push(ctx->objs, obj);
==> sk_X509_OBJECT_sort(ctx->objs); /* the call
X509_OBJECT_retrieve_match() will sort anyway, so
it's exactly one sort call more than before this
change, because a next call to X509_OBJECT_retrieve_match
will not sort any more, because the list is already
sorted */
}
the same fix should be done as well in the function X509_STORE_add_crl()
Please let me know if and how you will fix the bug.
With kind regards
Paolo
OpenSSL version: output of 'openssl version -a'
pa...@linux:~> /home/paolo/p/OpenSSL/spool/i486-linux-gcc3/bin/openssl version
-a
OpenSSL 0.9.8k 25 Mar 2009
built on: Tue Oct 27 10:00:38 CET 2009
platform: linux-elf-noefence
options: bn(64,32) md2(int) rc4(idx,int) des(ptr,risc1,16,long) idea(int)
blowfish(idx)
compiler: gcc -fPIC -DOPENSSL_PIC -DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN
-DHAVE_DLFCN_H -DDEBUG -DDEVRANDOM_CFGFILE="/etc/openssl.ini"
-DDEVRANDOM_LIB="/opt/adnseed/lib/libadnseed_so.so.1" -DBN_DEBUG -DREF_CHECK
-DCONF_DEBUG -DL_ENDIAN -DTERMIO -g -march=i486 -Wall
-DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_IA32_SSE2 -DSHA1_ASM -DMD5_ASM
-DRMD160_ASM -DAES_ASM
OPENSSLDIR: "/opt/openssl"
pa...@linux:~> uname -a
Linux linux 2.6.27.19-5-pae #1 SMP 2009-02-28 04:40:21 +0100 i686 i686 i386
GNU/Linux
pa...@linux:~> gcc -v
Using built-in specs.
Target: i586-suse-linux
Configured with: ../configure --prefix=/usr --infodir=/usr/share/info
--mandir=/usr/share/man --libdir=/usr/lib --libexecdir=/usr/lib
--enable-languages=c,c++,objc,fortran,obj-c++,java,ada
--enable-checking=release --with-gxx-include-dir=/usr/include/c++/4.3
--enable-ssp --disable-libssp --with-bugurl=http://bugs.opensuse.org/
--with-pkgversion='SUSE Linux' --disable-libgcj --disable-libmudflap
--with-slibdir=/lib --with-system-zlib --enable-__cxa_atexit
--enable-libstdcxx-allocator=new --disable-libstdcxx-pch
--enable-version-specific-runtime-libs --program-suffix=-4.3
--enable-linux-futex --without-system-libunwind --with-cpu=generic
--build=i586-suse-linux
Thread model: posix
gcc version 4.3.2 [gcc-4_3-branch revision 141291] (SUSE Linux)
--
AdNovum Informatik AG
Paolo Ganci, Senior Software Engineer
Dipl. El.-Ing. ETH
Roentgenstrasse 22, CH-8005 Zurich
mailto:[email protected]
phone: +41 44 272 6111, fax: +41 44 272 6312
http://www.adnovum.ch
AdNovum Locations: Bern, Budapest, Zurich (HQ)
______________________________________________________________________
OpenSSL Project http://www.openssl.org
Development Mailing List [email protected]
Automated List Manager [email protected]
