Hi, We are using OpenSSL 0.9.8r static library with novell cldapsdk. While doing SSL Bind and UnBind simultaneously from multi-threaded LDAP client application, its coring in OpenSSL library code path. Analyzed the cldapsdk code path, but couldn't find any obvious or suspected thing. It would be great help if somebody from OpenSSL team can provide some clarification on this issue.
Setup details:-
OS: SLES10 SP3
Arch: i686
OpenSSL version: 0.9.8r
Tried three different scenarios to get the clear call stack with OpenSSL debug
library.
1) export MALLOC_CHECK_=2
CMD: ./LDAPBind_i386 -h 127.0.0.1 -p 636 -D cn=admin,o=novell -w novell -b
o=novell -t 100 -o 10000 -l LDAPBind_Test.log -i 100
Run 1
(gdb) bt
#0 0xb7e81787 in sk_pop_free (st=0x805dbc0, func=0xb7e8c7af <X509_NAME_free>)
at stack.c:289
#1 0xb7e3d9c1 in SSL_CTX_free (a=0x80a2b58) at ssl_lib.c:1667
#2 0xb7e3ba0e in SSL_free (s=0x80d4688) at ssl_lib.c:528
#3 0xb7e324fe in iof_close (s=10, ld=0x0) at
../../../../../cldap/nldapx/ldapssl/src/ldapssl/ssl_io.c:797
#4 0xb7f78ed9 in ber_pvt_sb_close (sb=0x80be2f8) at sockbuf.c:660
#5 0xb7f87eb8 in ldap_close_connection (sb=0x80be2f8) at os-ip.c:336
#6 0xb7f8a01b in ldap_free_connection (ld=0x8146d38, lc=0x8120070, force=1,
unbind=1) at request.c:452
#7 0xb7f9f663 in ldap_ld_free (ld=0x8146d38, close=1, sctrls=0x0, cctrls=0x0)
at unbind.c:86
#8 0xb7f9f53f in ldap_unbind_ext (ld=0x8146d38, sctrls=0x0, cctrls=0x0) at
unbind.c:42
#9 0xb7f9f991 in ldap_unbind_s (ld=0x8146d38) at unbind.c:199
#10 0x080496e5 in ssl_user_ldapbind (tid=10) at src/LDAPBind.c:292
#11 0xb7f3b35b in start_thread () from /lib/libpthread.so.0
#12 0xb7d67c0e in clone () from /lib/libc.so.6
(gdb) quit
Run 2
(gdb) bt
#0 0xb7e4a787 in sk_pop_free (st=0x80febc8, func=0xb7e62908 <cleanup>) at
stack.c:289
#1 0xb7e629e3 in X509_STORE_free (vfy=0x8276160) at x509_lu.c:247
#2 0xb7e0693f in SSL_CTX_free (a=0x815b2e0) at ssl_lib.c:1659
#3 0xb7e04a0e in SSL_free (s=0x80d3d20) at ssl_lib.c:528
#4 0xb7dfb4fe in iof_close (s=20, ld=0x0) at
../../../../../cldap/nldapx/ldapssl/src/ldapssl/ssl_io.c:797
#5 0xb7f41ed9 in ber_pvt_sb_close (sb=0x8194928) at sockbuf.c:660
#6 0xb7f50eb8 in ldap_close_connection (sb=0x8194928) at os-ip.c:336
#7 0xb7f5301b in ldap_free_connection (ld=0x8057ad8, lc=0x815a290, force=1,
unbind=1) at request.c:452
#8 0xb7f68663 in ldap_ld_free (ld=0x8057ad8, close=1, sctrls=0x0, cctrls=0x0)
at unbind.c:86
#9 0xb7f6853f in ldap_unbind_ext (ld=0x8057ad8, sctrls=0x0, cctrls=0x0) at
unbind.c:42
#10 0xb7f68991 in ldap_unbind_s (ld=0x8057ad8) at unbind.c:199
#11 0x080496e5 in ssl_user_ldapbind (tid=13) at src/LDAPBind.c:292
#12 0xb7f0435b in start_thread () from /lib/libpthread.so.0
#13 0xb7d30c0e in clone () from /lib/libc.so.6
(gdb)
2) TCMalloc
CMD: LD_PRELOAD="/usr/lib/libtcmalloc.so.0.0.0" ./LDAPBind_i386 -h 127.0.0.1 -p
636 -D cn=admin,o=novell -w novell -b o=novell -t 100 -o 10000 -l
LDAPBind_Test.log -i 100
Run 1
(gdb) bt
#0 0xb7e2e5fc in CRYPTO_add_lock (pointer=0xb7ed9f18, amount=-1, type=3,
file=0xb7ee7d7c "tasn_utl.c", line=117)
at cryptlib.c:265
#1 0xb7e624e3 in asn1_do_lock (pval=0xb435afc0, op=-1, it=0xb7ee7ae4) at
tasn_utl.c:117
#2 0xb7e5f2f3 in asn1_item_combine_free (pval=0xb435afc0, it=0xb7ee7ae4,
combine=0) at tasn_fre.c:148
#3 0xb7e5f0bb in ASN1_item_free (val=0xb7ed9f08, it=0xb7ee7ae4) at
tasn_fre.c:71
#4 0xb7e5dfa8 in X509_free (a=0xb7ed9f08) at x_x509.c:136
#5 0xb7e6a927 in cleanup (a=0xb7ef7b20) at x509_lu.c:216
#6 0xb7e527a7 in sk_pop_free (st=0x848b6a8, func=0xb7e6a908 <cleanup>) at
stack.c:290
#7 0xb7e6a9e3 in X509_STORE_free (vfy=0x81f0780) at x509_lu.c:247
#8 0xb7e0e93f in SSL_CTX_free (a=0x81efdc0) at ssl_lib.c:1659
#9 0xb7e0ca0e in SSL_free (s=0x81efc80) at ssl_lib.c:528
#10 0xb7e034fe in iof_close (s=30, ld=0x0) at
../../../../../cldap/nldapx/ldapssl/src/ldapssl/ssl_io.c:797
#11 0xb7f49ed9 in ber_pvt_sb_close (sb=0x81f6e80) at sockbuf.c:660
#12 0xb7f58eb8 in ldap_close_connection (sb=0x81f6e80) at os-ip.c:336
#13 0xb7f5b01b in ldap_free_connection (ld=0x81fdf30, lc=0x81fa4b0, force=1,
unbind=1) at request.c:452
#14 0xb7f70663 in ldap_ld_free (ld=0x81fdf30, close=1, sctrls=0x0, cctrls=0x0)
at unbind.c:86
#15 0xb7f7053f in ldap_unbind_ext (ld=0x81fdf30, sctrls=0x0, cctrls=0x0) at
unbind.c:42
#16 0xb7f70991 in ldap_unbind_s (ld=0x81fdf30) at unbind.c:199
#17 0x080496e5 in ssl_user_ldapbind (tid=8) at src/LDAPBind.c:292
#18 0xb7f0b35b in start_thread () from /lib/libpthread.so.0
#19 0xb7d38c0e in clone () from /lib/libc.so.6
(gdb)
Run 2
(gdb) bt
#0 0xb7e3e5fc in CRYPTO_add_lock (pointer=0xb7ee9f18, amount=-1, type=3,
file=0xb7ef7d7c "tasn_utl.c", line=117)
at cryptlib.c:265
#1 0xb7e724e3 in asn1_do_lock (pval=0xb436afc0, op=-1, it=0xb7ef7ae4) at
tasn_utl.c:117
#2 0xb7e6f2f3 in asn1_item_combine_free (pval=0xb436afc0, it=0xb7ef7ae4,
combine=0) at tasn_fre.c:148
#3 0xb7e6f0bb in ASN1_item_free (val=0xb7ee9f08, it=0xb7ef7ae4) at
tasn_fre.c:71
#4 0xb7e6dfa8 in X509_free (a=0xb7ee9f08) at x_x509.c:136
#5 0xb7e7a927 in cleanup (a=0xb7f07b20) at x509_lu.c:216
#6 0xb7e627a7 in sk_pop_free (st=0x8591ed0, func=0xb7e7a908 <cleanup>) at
stack.c:290
#7 0xb7e7a9e3 in X509_STORE_free (vfy=0x81fc7c0) at x509_lu.c:247
#8 0xb7e1e93f in SSL_CTX_free (a=0x81fbc80) at ssl_lib.c:1659
#9 0xb7e1ca0e in SSL_free (s=0x81fbdc0) at ssl_lib.c:528
#10 0xb7e134fe in iof_close (s=53, ld=0x0) at
../../../../../cldap/nldapx/ldapssl/src/ldapssl/ssl_io.c:797
#11 0xb7f59ed9 in ber_pvt_sb_close (sb=0x8202f80) at sockbuf.c:660
#12 0xb7f68eb8 in ldap_close_connection (sb=0x8202f80) at os-ip.c:336
#13 0xb7f6b01b in ldap_free_connection (ld=0x8209f30, lc=0x82063e8, force=1,
unbind=1) at request.c:452
#14 0xb7f80663 in ldap_ld_free (ld=0x8209f30, close=1, sctrls=0x0, cctrls=0x0)
at unbind.c:86
#15 0xb7f8053f in ldap_unbind_ext (ld=0x8209f30, sctrls=0x0, cctrls=0x0) at
unbind.c:42
#16 0xb7f80991 in ldap_unbind_s (ld=0x8209f30) at unbind.c:199
#17 0x080496e5 in ssl_user_ldapbind (tid=8) at src/LDAPBind.c:292
#18 0xb7f1b35b in start_thread () from /lib/libpthread.so.0
#19 0xb7d48c0e in clone () from /lib/libc.so.6
(gdb)
3) Valgrind
CMD: valgrind --tool=memcheck --show-reachable=yes --leak-resolution=high
--leak-check=yes --show-reachable=yes --log-file=valgrind_memcheck.log
--num-callers=25 --error-limit=no ./LDAPBind_i386 -h 127.0.0.1 -p 636 -D
cn=admin,o=novell -w novell -b o=novell -t 1000 -o 10000 -l LDAPBind_Test.log
-i 100
Run 1
==32434== Thread 242:
==32434== Invalid read of size 4
==32434== at 0x416AC3F: SHA1_Update (md32_common.h:291)
==32434== by 0x414E760: update (m_sha1.c:77)
==32434== by 0x414BD41: EVP_DigestUpdate (digest.c:325)
==32434== by 0x4196DA5: ssleay_rand_bytes (md_rand.c:473)
==32434== by 0x414A4D6: RAND_bytes (rand_lib.c:227)
==32434== by 0x411E11F: ssl3_send_client_key_exchange (s3_clnt.c:1923)
==32434== by 0x411AF08: ssl3_connect (s3_clnt.c:361)
==32434== by 0x410438A: SSL_connect (ssl_lib.c:879)
==32434== by 0x40FF2F6: ssl23_get_server_hello (s23_clnt.c:617)
==32434== by 0x40FE5A6: ssl23_connect (s23_clnt.c:169)
==32434== by 0x410438A: SSL_connect (ssl_lib.c:879)
==32434== by 0x40FA3AE: iof_connect (ssl_io.c:721)
==32434== by 0x4056BC8: ldap_pvt_connect (os-ip.c:230)
==32434== by 0x4056E18: ldap_connect_to_host (os-ip.c:317)
==32434== by 0x40565F4: open_ldap_connection (open.c:374)
==32434== by 0x4058B7C: ldap_new_connection (request.c:312)
==32434== by 0x4055FCF: ldap_open_defconn (open.c:102)
==32434== by 0x405856D: ldap_send_initial_request (request.c:114)
==32434== by 0x405CFC3: ldap_sasl_bind (sasl.c:223)
==32434== by 0x405D0E9: ldap_sasl_bind_s (sasl.c:269)
==32434== by 0x405D694: ldap_simple_bind_s (sbind.c:113)
==32434== by 0x804967C: ssl_user_ldapbind (LDAPBind.c:285)
==32434== by 0x409035A: start_thread (in /lib/libpthread-2.4.so)
==32434== by 0x42C1BDD: clone (in /lib/libc-2.4.so)
==32434== Address 0x14 is not stack'd, malloc'd or (recently) free'd
==32434==
==32434==
==32434== Process terminating with default action of signal 11 (SIGSEGV):
dumping core
==32434== Access not within mapped region at address 0x14
==32434== at 0x416AC3F: SHA1_Update (md32_common.h:291)
==32434== by 0x414E760: update (m_sha1.c:77)
==32434== by 0x414BD41: EVP_DigestUpdate (digest.c:325)
==32434== by 0x4196DA5: ssleay_rand_bytes (md_rand.c:473)
==32434== by 0x414A4D6: RAND_bytes (rand_lib.c:227)
==32434== by 0x411E11F: ssl3_send_client_key_exchange (s3_clnt.c:1923)
==32434== by 0x411AF08: ssl3_connect (s3_clnt.c:361)
==32434== by 0x410438A: SSL_connect (ssl_lib.c:879)
==32434== by 0x40FF2F6: ssl23_get_server_hello (s23_clnt.c:617)
==32434== by 0x40FE5A6: ssl23_connect (s23_clnt.c:169)
==32434== by 0x410438A: SSL_connect (ssl_lib.c:879)
==32434== by 0x40FA3AE: iof_connect (ssl_io.c:721)
==32434== by 0x4056BC8: ldap_pvt_connect (os-ip.c:230)
==32434== by 0x4056E18: ldap_connect_to_host (os-ip.c:317)
==32434== by 0x40565F4: open_ldap_connection (open.c:374)
==32434== by 0x4058B7C: ldap_new_connection (request.c:312)
==32434== by 0x4055FCF: ldap_open_defconn (open.c:102)
==32434== by 0x405856D: ldap_send_initial_request (request.c:114)
==32434== by 0x405CFC3: ldap_sasl_bind (sasl.c:223)
==32434== by 0x405D0E9: ldap_sasl_bind_s (sasl.c:269)
==32434== by 0x405D694: ldap_simple_bind_s (sbind.c:113)
==32434== by 0x804967C: ssl_user_ldapbind (LDAPBind.c:285)
==32434== by 0x409035A: start_thread (in /lib/libpthread-2.4.so)
==32434== by 0x42C1BDD: clone (in /lib/libc-2.4.so)
==32434== If you believe this happened as a result of a stack
==32434== overflow in your program's main thread (unlikely but
==32434== possible), you can try to increase the size of the
==32434== main thread stack using the --main-stacksize= flag.
==32434== The main thread stack size used in this run was 8388608.
valgrind: m_coredump/coredump-elf.c:597 (make_elf_coredump): Assertion
'VG_(lseek)(core_fd, phdrs[idx].p_offset, VKI_SEEK_SET) == phdrs[idx].p_offset'
failed.
==32434== at 0x38029298: report_and_quit (m_libcassert.c:193)
==32434== by 0x3802946C: vgPlain_assert_fail (m_libcassert.c:267)
==32434== by 0x38050827: vgPlain_make_coredump (coredump-elf.c:597)
==32434== by 0x3803C462: deliver_signal (m_signals.c:1647)
==32434== by 0x3803DCB1: sync_signalhandler (m_signals.c:2306)
==32434== by 0x3803B4FF: ??? (in /usr/local/lib/valgrind/memcheck-x86-linux)
Run 2
==950== Thread 53:
==950== Invalid read of size 4
==950== at 0x416AC3F: SHA1_Update (md32_common.h:291)
==950== by 0x414E760: update (m_sha1.c:77)
==950== by 0x414BD41: EVP_DigestUpdate (digest.c:325)
==950== by 0x4196DA5: ssleay_rand_bytes (md_rand.c:473)
==950== by 0x414A4D6: RAND_bytes (rand_lib.c:227)
==950== by 0x411E11F: ssl3_send_client_key_exchange (s3_clnt.c:1923)
==950== by 0x411AF08: ssl3_connect (s3_clnt.c:361)
==950== by 0x410438A: SSL_connect (ssl_lib.c:879)
==950== by 0x40FF2F6: ssl23_get_server_hello (s23_clnt.c:617)
==950== by 0x40FE5A6: ssl23_connect (s23_clnt.c:169)
==950== by 0x410438A: SSL_connect (ssl_lib.c:879)
==950== by 0x40FA3AE: iof_connect (ssl_io.c:721)
==950== by 0x4056BC8: ldap_pvt_connect (os-ip.c:230)
==950== by 0x4056E18: ldap_connect_to_host (os-ip.c:317)
==950== by 0x40565F4: open_ldap_connection (open.c:374)
==950== by 0x4058B7C: ldap_new_connection (request.c:312)
==950== by 0x4055FCF: ldap_open_defconn (open.c:102)
==950== by 0x405856D: ldap_send_initial_request (request.c:114)
==950== by 0x405CFC3: ldap_sasl_bind (sasl.c:223)
==950== by 0x405D0E9: ldap_sasl_bind_s (sasl.c:269)
==950== by 0x405D694: ldap_simple_bind_s (sbind.c:113)
==950== by 0x804967C: ssl_user_ldapbind (LDAPBind.c:285)
==950== by 0x409035A: start_thread (in /lib/libpthread-2.4.so)
==950== by 0x42C1BDD: clone (in /lib/libc-2.4.so)
==950== Address 0x14 is not stack'd, malloc'd or (recently) free'd
==950==
==950==
==950== Process terminating with default action of signal 11 (SIGSEGV): dumping
core
==950== Access not within mapped region at address 0x14
==950== at 0x416AC3F: SHA1_Update (md32_common.h:291)
==950== by 0x414E760: update (m_sha1.c:77)
==950== by 0x414BD41: EVP_DigestUpdate (digest.c:325)
==950== by 0x4196DA5: ssleay_rand_bytes (md_rand.c:473)
==950== by 0x414A4D6: RAND_bytes (rand_lib.c:227)
==950== by 0x411E11F: ssl3_send_client_key_exchange (s3_clnt.c:1923)
==950== by 0x411AF08: ssl3_connect (s3_clnt.c:361)
==950== by 0x410438A: SSL_connect (ssl_lib.c:879)
==950== by 0x40FF2F6: ssl23_get_server_hello (s23_clnt.c:617)
==950== by 0x40FE5A6: ssl23_connect (s23_clnt.c:169)
==950== by 0x410438A: SSL_connect (ssl_lib.c:879)
==950== by 0x40FA3AE: iof_connect (ssl_io.c:721)
==950== by 0x4056BC8: ldap_pvt_connect (os-ip.c:230)
==950== by 0x4056E18: ldap_connect_to_host (os-ip.c:317)
==950== by 0x40565F4: open_ldap_connection (open.c:374)
==950== by 0x4058B7C: ldap_new_connection (request.c:312)
==950== by 0x4055FCF: ldap_open_defconn (open.c:102)
==950== by 0x405856D: ldap_send_initial_request (request.c:114)
==950== by 0x405CFC3: ldap_sasl_bind (sasl.c:223)
==950== by 0x405D0E9: ldap_sasl_bind_s (sasl.c:269)
==950== by 0x405D694: ldap_simple_bind_s (sbind.c:113)
==950== by 0x804967C: ssl_user_ldapbind (LDAPBind.c:285)
==950== by 0x409035A: start_thread (in /lib/libpthread-2.4.so)
==950== by 0x42C1BDD: clone (in /lib/libc-2.4.so)
==950== If you believe this happened as a result of a stack
==950== overflow in your program's main thread (unlikely but
==950== possible), you can try to increase the size of the
==950== main thread stack using the --main-stacksize= flag.
==950== The main thread stack size used in this run was 8388608.
valgrind: m_coredump/coredump-elf.c:597 (make_elf_coredump): Assertion
'VG_(lseek)(core_fd, phdrs[idx].p_offset, VKI_SEEK_SET) == phdrs[idx].p_offset'
failed.
==950== at 0x38029298: report_and_quit (m_libcassert.c:193)
==950== by 0x3802946C: vgPlain_assert_fail (m_libcassert.c:267)
==950== by 0x38050827: vgPlain_make_coredump (coredump-elf.c:597)
==950== by 0x3803C462: deliver_signal (m_signals.c:1647)
==950== by 0x3803DCB1: sync_signalhandler (m_signals.c:2306)
==950== by 0x3803B4FF: ??? (in /usr/local/lib/valgrind/memcheck-x86-linux)
Please let me know if you need any more details.
Attached: LDAPBind source(LDAPBind.c) and bin(LDAPBind_i386) files
Regards,
Sandip
LDAPBind.C
Description: Binary data
LDAPBind_i386
Description: Binary data
