Dear Dmitry, Thank you for your response. Here is the stack trace ============================================================================================================================= $ strace openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out msg.enc -engine rsa-engine-new execve("/usr/bin/openssl", ["openssl", "rsautl", "-encrypt", "-inkey", "public.pem", "-pubin", "-in", "msg.txt", "-out", "msg.enc", "-engine", "rsa-engine-new"], [/* 65 vars */]) = 0 brk(NULL) = 0x1634000 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=103650, ...}) = 0 mmap(NULL, 103650, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3997ce3000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/opt/openssl_test/lib/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\327\1\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=1713040, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce2000 mmap(NULL, 2836448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3997823000 mprotect(0x7f39978cb000, 2097152, PROT_NONE) = 0 mmap(0x7f3997acb000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa8000) = 0x7f3997acb000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/opt/openssl_test/lib/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0p\7\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=6286208, ...}) = 0 mmap(NULL, 5508104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39972e2000 mprotect(0x7f39975f3000, 2093056, PROT_NONE) = 0 mmap(0x7f39977f2000, 188416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x310000) = 0x7f39977f2000 mmap(0x7f3997820000, 11272, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3997820000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0 mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39970c5000 mprotect(0x7f39970dd000, 2093056, PROT_NONE) = 0 mmap(0x7f39972dc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f39972dc000 mmap(0x7f39972de000, 13352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39972de000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\t\2\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce1000 mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3996cfb000 mprotect(0x7f3996ebb000, 2097152, PROT_NONE) = 0 mmap(0x7f39970bb000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7f39970bb000 mmap(0x7f39970c1000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39970c1000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0 mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3996af7000 mprotect(0x7f3996afa000, 2093056, PROT_NONE) = 0 mmap(0x7f3996cf9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3996cf9000 close(3) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce0000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997cdf000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997cde000 arch_prctl(ARCH_SET_FS, 0x7f3997cdf700) = 0 mprotect(0x7f39970bb000, 16384, PROT_READ) = 0 mprotect(0x7f3996cf9000, 4096, PROT_READ) = 0 mprotect(0x7f39972dc000, 4096, PROT_READ) = 0 mprotect(0x7f39977f2000, 176128, PROT_READ) = 0 mprotect(0x7f3997acb000, 32768, PROT_READ) = 0 mprotect(0x6ac000, 4096, PROT_READ) = 0 mprotect(0x7f3997cfd000, 4096, PROT_READ) = 0 munmap(0x7f3997ce3000, 103650) = 0 set_tid_address(0x7f3997cdf9d0) = 6006 set_robust_list(0x7f3997cdf9e0, 24) = 0 rt_sigaction(SIGRTMIN, {0x7f39970cab50, [], SA_RESTORER|SA_SIGINFO, 0x7f39970d6390}, NULL, 8) = 0 rt_sigaction(SIGRT_1, {0x7f39970cabe0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f39970d6390}, NULL, 8) = 0 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0 brk(NULL) = 0x1634000 brk(0x1655000) = 0x1655000 futex(0x7f39978213d8, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f39978213e4, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f39978213ec, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997821368, FUTEX_WAKE_PRIVATE, 2147483647) = 0 rt_sigaction(SIGPIPE, {SIG_IGN, [PIPE], SA_RESTORER|SA_RESTART, 0x7f3996d304c0}, {SIG_DFL, [], 0}, 8) = 0 futex(0x7f3997821530, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997821400, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997821408, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f399781f880, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f399781f940, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f399781f94c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f39978213f4, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997821438, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997821440, FUTEX_WAKE_PRIVATE, 2147483647) = 0 open("/opt/openssl_test/ssl/openssl.cnf", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=11162, ...}) = 0 read(3, "#\n# OpenSSL example configuratio"..., 4096) = 4096 read(3, "\n# MASK:XXXX a literal mask valu"..., 4096) = 4096 read(3, "ts=CA:FALSE\n\n# Here are some exa"..., 4096) = 2970 brk(0x1676000) = 0x1676000 read(3, "", 4096) = 0 close(3) = 0 futex(0x7f3997821410, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3997ad7698, FUTEX_WAKE_PRIVATE, 2147483647) = 0 sysinfo({uptime=2961, loads=[30016, 30688, 27392], totalram=16689545216, freeram=12341272576, sharedram=140345344, bufferram=240164864, totalswap=1023406080, freeswap=1023406080, procs=922, totalhigh=0, freehigh=0, mem_unit=1}) = 0 futex(0x7f3997ad7778, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x7f3996cfa0a8, FUTEX_WAKE_PRIVATE, 2147483647) = 0 open("/opt/openssl_test/lib/engines-1.1/rsa-engine-new.so", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p0\0\0\0\0\0\0"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=143576, ...}) = 0 mmap(NULL, 2241392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39968d3000 mprotect(0x7f39968ea000, 2093056, PROT_NONE) = 0 mmap(0x7f3996ae9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3996ae9000 mmap(0x7f3996aeb000, 45936, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3996aeb000 close(3) = 0 mprotect(0x7f3996ae9000, 4096, PROT_READ) = 0 write(2, "engine \"rsa-engine-new\" set.\n", 29engine "rsa-engine-new" set. ) = 29 open("public.pem", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0664, st_size=451, ...}) = 0 read(3, "-----BEGIN PUBLIC KEY-----\nMIIBI"..., 4096) = 451 brk(0x1674000) = 0x1674000 close(3) = 0 open("msg.txt", O_RDONLY) = 3 open("msg.enc", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 4 fstat(3, {st_mode=S_IFREG|0664, st_size=72, ...}) = 0 read(3, "Hi. This is a test message. Hope"..., 4096) = 72 read(3, "", 4096) = 0 --- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0} --- +++ killed by SIGSEGV (core dumped) +++ Segmentation fault (core dumped) =============================================================================================================================
I was trying using gdb to debug the error. I get to until RSA_public_encrypt(), then if I step in, I get the segmentation fault, ================================================================================================== bt #0 0x0000000000000000 in ?? () #1 0x00007ffff77dbfc0 in RSA_public_encrypt (flen=72, from=0x6d8860 "Hi. This is a test message. Hope to see some performace gain with this.\nme", to=0x6d8a70 "x\353;\367\377\177", rsa=0x6d8540, padding=1) at crypto/rsa/rsa_crpt.c:30 #2 0x00000000004479a7 in rsautl_main (argc=0, argv=0x7fffffffde10) at apps/rsautl.c:248 #3 0x00000000004379fa in do_cmd (prog=0x6d5930, argc=11, argv=0x7fffffffde10) at apps/openssl.c:564 #4 0x0000000000436e4d in main (argc=11, argv=0x7fffffffde10) at apps/openssl.c:183 ================================================================================================== To my understanding, instead of pointing to the default encryption function, it is pointing to 0x0000000000000000, thus causing the segmentation fault. Thanks, Shariful On Tue, Sep 28, 2021 at 1:40 AM Dmitry Belyavsky <beld...@gmail.com> wrote: > Dear Shariful, > > Could you please also provide a stack trace of your segfault? > > On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto...@gmail.com> wrote: > >> Hello, >> I have the following simple RSA engine code from *e_dasync.c. * Following >> code compiles and works. Until now, I was under the impression that if I do >> not supply an alternative method in function *bind_dasync(), the *engine >> will use the default method. However, it doesn't seem to be the case. If I >> comment out line 37 and try to perform an encryption operation with the >> following command, >> >> *"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out >> msg.enc -engine rsa-engine-new" * >> >> I get segmentation errors. >> >> Can anyone please tell me why this is happening? >> >> >> ============================================================================== >> >> 1. /* Engine Id and Name */ >> 2. static const char *engine_rsa_id = "rsa-engine-new"; >> 3. static const char *engine_rsa_name = "RSA engine for testing"; >> 4. >> 5. // data encryption function >> 6. static int eng_rsa_pub_enc(int flen, const unsigned char *from, >> 7. unsigned char *to, RSA *rsa, int >> padding) { >> 8. printf("Encryption\n"); >> 9. return 0; >> 10. } >> 11. >> 12. // signature verify >> 13. static int eng_rsa_pub_dec(int flen, const unsigned char *from, >> unsigned char *to, RSA *rsa, int padding){ >> 14. printf("Signature verify:\n"); >> 15. return 0; >> 16. } >> 17. >> 18. // signature >> 19. static int eng_rsa_priv_enc(int flen, const unsigned char *from, >> unsigned char *to, RSA *rsa, int padding){ >> 20. printf("Signature:\n"); >> 21. return 0; >> 22. } >> 23. >> 24. // data decryption >> 25. static int eng_rsa_priv_dec(int flen, const unsigned char *from, >> unsigned char *to, RSA *rsa, int padding){ >> 26. printf("Decryption\n"); >> 27. return 0; >> 28. } >> 29. >> 30. >> 31. static RSA_METHOD *test_rsa_method = NULL; >> 32. >> 33. >> 34. static int bind_dasync(ENGINE *e){ >> 35. /* Setup RSA_METHOD */ >> 36. if ((test_rsa_method = RSA_meth_new("Test RSA method", 0)) == >> NULL >> 37. // || RSA_meth_set_pub_enc(test_rsa_method, >> eng_rsa_pub_enc) == 0 >> 38. || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec) >> == 0 >> 39. || RSA_meth_set_priv_enc(test_rsa_method, >> eng_rsa_priv_enc) == 0 >> 40. || RSA_meth_set_priv_dec(test_rsa_method, >> eng_rsa_priv_dec) == 0 >> 41. ) { >> 42. >> 43. return 0; >> 44. } >> 45. >> 46. /* Ensure the dasync error handling is set up */ >> 47. >> 48. if (!ENGINE_set_id(e, engine_rsa_id) >> 49. || !ENGINE_set_name(e, engine_rsa_name) >> 50. || !ENGINE_set_RSA(e, test_rsa_method) >> 51. ) { >> 52. return 0; >> 53. } >> 54. return 1; >> 55. } >> 56. >> 57. static int bind_helper(ENGINE *e, const char *id){ >> 58. if (!bind_dasync(e)){ >> 59. printf("2_Error: Inside Bind helper\n"); >> 60. return 0; >> 61. } >> 62. return 1; >> 63. } >> 64. >> 65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper) >> 66. IMPLEMENT_DYNAMIC_CHECK_FN() >> >> >> ============================================================================== >> >> Regards, >> Shariful Alam >> >> > > -- > SY, Dmitry Belyavsky >