Hello, just tried to get some more information where this stack smashing happens.
The stack canary gets at least overwritten here: 165 memmove(detect + n + 1, detect + n, 166 (j - n) * sizeof(int)); The variable detect has a size of 12. The parameter to memove is the second element, therefore the size parameter to memove should be <= 8. But actually the size parameter to memove is 12, therefore memove writes beyond the variable detect. Attached file shows some debugging attempts. Kind regards, Bernhard
# 2023-07-11 bookworm/stable amd64 qemu VM apt update apt dist-upgrade apt install systemd-coredump gdb rr valgrind outguess outguess-dbgsym apt build-dep outguess mkdir /home/benutzer/source/outguess/orig -p cd /home/benutzer/source/outguess/orig apt source outguess benutzer@debian:~$ wget -q https://upload.wikimedia.org/wikipedia/commons/3/3f/JPEG_example_flower.jpg benutzer@debian:~$ echo msg1 > msg1.txt benutzer@debian:~$ echo msg2 > msg2.txt benutzer@debian:~$ outguess -k "key1" -d msg1.txt -E -K "key2" -D msg2.txt -p 100 JPEG_example_flower.jpg JPEG_example_flower.steg.jpg Initialize encoding/decoding tables Reading JPEG_example_flower.jpg.... JPEG compression quality set to 100 Extracting usable bits: 70325 bits Correctable message size: 17434 bits, 24.79% Encoded 'msg1.txt': 40 bits, 5 bytes Finding best embedding... 0: 33(45.8%)[82.5%], bias 28(0.85), saved: -1, total: 0.05% 1: 28(38.9%)[70.0%], bias 25(0.89), saved: -1, total: 0.04% 6: 30(42.3%)[75.0%], bias 19(0.63), saved: -1, total: 0.04% 11: 28(38.9%)[70.0%], bias 13(0.46), saved: -1, total: 0.04% 11, 41: Embedding data: 40 in 70325 Bits embedded: 72, changed: 28(38.9%)[70.0%], bias: 13, tot: 68673, skip: 68601 Encoded 'msg2.txt' with ECC: 96 bits, 12 bytes Finding best embedding... *** stack smashing detected ***: terminated Abgebrochen (Speicherabzug geschrieben) cd /home/benutzer/source/outguess/orig/outguess-0.4 coredumpctl list TIME PID UID GID SIG COREFILE EXE SIZE Tue 2023-07-11 10:00:02 CEST 1136 1000 1000 SIGABRT present /usr/bin/outguess 412.4K coredumpctl gdb --debugger-argument=-q 1136 (gdb) bt #0 __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44 #1 0x00007f83ca05bd2f in __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78 #2 0x00007f83ca00cef2 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26 #3 0x00007f83c9ff7472 in __GI_abort () at ./stdlib/abort.c:79 #4 0x00007f83ca0502d0 in __libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7f83ca16a210 "*** %s ***: terminated\n") at ../sysdeps/posix/libc_fatal.c:155 #5 0x00007f83ca0e8e82 in __GI___fortify_fail (msg=msg@entry=0x7f83ca16a1f8 "stack smashing detected") at ./debug/fortify_fail.c:26 #6 0x00007f83ca0e8e60 in __stack_chk_fail () at ./debug/stack_chk_fail.c:24 #7 0x00005627c3781422 in steg_adjust_errors (bitmap=bitmap@entry=0x7ffda38f0c30, flags=flags@entry=8) at ./src/outguess.c:187 #8 0x00005627c37814ba in steg_embedchunk (bitmap=bitmap@entry=0x7ffda38f0c30, iter=iter@entry=0x7ffda38f0670, data=0, bits=1, bits@entry=8, embed=embed@entry=8) at ./src/outguess.c:206 #9 0x00005627c3781aa8 in steg_embed (bitmap=bitmap@entry=0x7ffda38f0c30, iter=iter@entry=0x7ffda38f0670, as=as@entry=0x7ffda38f0560, data=data@entry=0x5627c4d96b30 "BU\026\330E\315>ۆL\260\246\203\177", datalen=datalen@entry=12, seed=seed@entry=0, embed=8) at ./src/outguess.c:285 #10 0x00005627c3781e44 in steg_find (bitmap=bitmap@entry=0x7ffda38f0c30, iter=iter@entry=0x7ffda38f0a30, as=as@entry=0x7ffda38f0810, siter=<optimized out>, siterstart=<optimized out>, data=data@entry=0x5627c4d96b30 "BU\026\330E\315>ۆL\260\246\203\177", datalen=12, flags=8) at ./src/outguess.c:444 #11 0x00005627c3782a04 in do_embed (bitmap=bitmap@entry=0x7ffda38f0c30, filename=filename@entry=0x7ffda38f16ec "msg2.txt", key=key@entry=0x7ffda38f0fe0 "key2", klen=<optimized out>, cfg=cfg@entry=0x7ffda38f0c24, result=result@entry=0x7ffda38f0c08) at ./src/outguess.c:719 #12 0x00005627c3780d12 in main (argc=<optimized out>, argv=0x7ffda38f1488) at ./src/outguess.c:1025 (gdb) up #7 0x00005627c3781422 in steg_adjust_errors (bitmap=bitmap@entry=0x7ffda38f0c30, flags=flags@entry=8) at ./src/outguess.c:187 187 } (gdb) list 138, 188 138 void 139 steg_adjust_errors(bitmap *bitmap, int flags) 140 { 141 int i, j, n, many, flag; 142 int priority[ERRORBITS], detect[ERRORBITS]; 143 144 many = ERRORBITS - steg_errors; 145 for (j = 0; j < many && j < steg_err_cnt; j++) { 146 priority[j] = steg_err_buf[j]; 147 detect[j] = bitmap->detect[priority[j]]; 148 } 149 150 /* Very simple sort */ 151 do { 152 for (flag = 0, i = 0; i < j - 1; i++) 153 if (detect[i] < detect[i + 1]) { 154 SWAP(detect[i], detect[i+1]); 155 SWAP(priority[i], priority[i+1]); 156 flag = 1; 157 } 158 } while (flag); 159 160 for (i = j; i < steg_err_cnt; i++) { 161 for (n = 0; n < j; n++) 162 if (detect[n] < bitmap->detect[steg_err_buf[i]]) 163 break; 164 if (n < j - 1) { 165 memmove(detect + n + 1, detect + n, 166 (j - n) * sizeof(int)); 167 memmove(priority + n + 1, priority + n, 168 (j - n) * sizeof(int)); 169 } 170 if (n < j) { 171 priority[n] = steg_err_buf[i]; 172 detect[n] = bitmap->detect[steg_err_buf[i]]; 173 } 174 } 175 176 for (i = 0; i < j; i++) { 177 if (flags & STEG_EMBED) { 178 WRITE_BIT(bitmap->locked, i, 0); 179 if (TEST_BIT(bitmap->bitmap, priority[i])) 180 WRITE_BIT(bitmap->bitmap, i, 0); 181 else 182 WRITE_BIT(bitmap->bitmap, i, 1); 183 } 184 steg_mis--; 185 steg_mod -= detect[i]; 186 } 187 } 188 benutzer@debian:~$ valgrind outguess -k "key1" -d msg1.txt -E -K "key2" -D msg2.txt -p 100 JPEG_example_flower.jpg JPEG_example_flower.steg.jpg ==5915== Memcheck, a memory error detector ==5915== Copyright (C) 2002-2022, and GNU GPL'd, by Julian Seward et al. ==5915== Using Valgrind-3.19.0 and LibVEX; rerun with -h for copyright info ==5915== Command: outguess -k key1 -d msg1.txt -E -K key2 -D msg2.txt -p 100 JPEG_example_flower.jpg JPEG_example_flower.steg.jpg ==5915== Initialize encoding/decoding tables Reading JPEG_example_flower.jpg.... JPEG compression quality set to 100 Extracting usable bits: 70325 bits Correctable message size: 17434 bits, 24.79% Encoded 'msg1.txt': 40 bits, 5 bytes Finding best embedding... 0: 33(45.8%)[82.5%], bias 28(0.85), saved: -1, total: 0.05% 1: 28(38.9%)[70.0%], bias 25(0.89), saved: -1, total: 0.04% 6: 30(42.3%)[75.0%], bias 19(0.63), saved: -1, total: 0.04% 11: 28(38.9%)[70.0%], bias 13(0.46), saved: -1, total: 0.04% 11, 41: Embedding data: 40 in 70325 Bits embedded: 72, changed: 28(38.9%)[70.0%], bias: 13, tot: 68673, skip: 68601 Encoded 'msg2.txt' with ECC: 96 bits, 12 bytes Finding best embedding... *** stack smashing detected ***: terminated ==5915== ==5915== Process terminating with default action of signal 6 (SIGABRT) ==5915== at 0x49BECCC: __pthread_kill_implementation (pthread_kill.c:44) ==5915== by 0x496FEF1: raise (raise.c:26) ==5915== by 0x495A471: abort (abort.c:79) ==5915== by 0x49B32CF: __libc_message (libc_fatal.c:155) ==5915== by 0x4A4BE81: __fortify_fail (fortify_fail.c:26) ==5915== by 0x4A4BE5F: __stack_chk_fail (stack_chk_fail.c:24) ==5915== by 0x10B421: steg_adjust_errors (outguess.c:187) ==5915== by 0x10B4B9: steg_embedchunk (outguess.c:206) ==5915== by 0x10BAA7: steg_embed (outguess.c:285) ==5915== by 0x10BE43: steg_find (outguess.c:444) ==5915== by 0x10CA03: do_embed (outguess.c:719) ==5915== by 0x10AD11: main (outguess.c:1025) ==5915== ==5915== HEAP SUMMARY: ==5915== in use at exit: 891,416 bytes in 15 blocks ==5915== total heap usage: 411 allocs, 396 frees, 3,261,833 bytes allocated ==5915== ==5915== LEAK SUMMARY: ==5915== definitely lost: 2,048 bytes in 3 blocks ==5915== indirectly lost: 0 bytes in 0 blocks ==5915== possibly lost: 0 bytes in 0 blocks ==5915== still reachable: 889,368 bytes in 12 blocks ==5915== suppressed: 0 bytes in 0 blocks ==5915== Rerun with --leak-check=full to see details of leaked memory ==5915== ==5915== For lists of detected and suppressed errors, rerun with: -s ==5915== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) Abgebrochen (Speicherabzug geschrieben) benutzer@debian:~$ rr record outguess -k "key1" -d msg1.txt -E -K "key2" -D msg2.txt -p 100 JPEG_example_flower.jpg JPEG_example_flower.steg.jpg rr: Saving execution to trace directory `/home/benutzer/.local/share/rr/outguess-0'. Initialize encoding/decoding tables Reading JPEG_example_flower.jpg.... JPEG compression quality set to 100 Extracting usable bits: 70325 bits Correctable message size: 17434 bits, 24.79% Encoded 'msg1.txt': 40 bits, 5 bytes Finding best embedding... 0: 33(45.8%)[82.5%], bias 28(0.85), saved: -1, total: 0.05% 1: 28(38.9%)[70.0%], bias 25(0.89), saved: -1, total: 0.04% 6: 30(42.3%)[75.0%], bias 19(0.63), saved: -1, total: 0.04% 11: 28(38.9%)[70.0%], bias 13(0.46), saved: -1, total: 0.04% 11, 41: Embedding data: 40 in 70325 Bits embedded: 72, changed: 28(38.9%)[70.0%], bias: 13, tot: 68673, skip: 68601 Encoded 'msg2.txt' with ECC: 96 bits, 12 bytes Finding best embedding... *** stack smashing detected ***: terminated Abgebrochen cd /home/benutzer/source/outguess/orig/outguess-0.4 rr replay --debugger-option=-q outguess-0 set width 0 set pagination off cont b steg_adjust_errors reverse-cont reverse-cont display/i $pc (rr) stepi 0x000055771c8b112d 140 { 1: x/i $pc => 0x55771c8b112d <steg_adjust_errors+45>: mov %rax,0x58(%rsp) (rr) x/1xg $rsp + 0x58 0x7ffc6a268be8: 0x00007fc5aac7289a (rr) stepi 0x000055771c8b1132 140 { 1: x/i $pc => 0x55771c8b1132 <steg_adjust_errors+50>: xor %eax,%eax (rr) x/1xg $rsp + 0x58 0x7ffc6a268be8: 0x3462b133551b0000 (rr) watch *0x7ffc6a268be8 Hardware watchpoint 2: *0x7ffc6a268be8 (rr) cont Continuing. Hardware watchpoint 2: *0x7ffc6a268be8 Old value = 1427832832 New value = 0 __memcpy_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:376 376 ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S: Datei oder Verzeichnis nicht gefunden. 1: x/i $pc => 0x7fc5aad2c610 <__memcpy_avx_unaligned_erms+144>: ret (rr) bt #0 __memcpy_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:376 #1 0x000055771c8b1301 in memmove (__len=12, __src=<optimized out>, __dest=0x7ffc6a268be0) at /usr/include/x86_64-linux-gnu/bits/string_fortified.h:36 #2 steg_adjust_errors (bitmap=bitmap@entry=0x7ffc6a269470, flags=flags@entry=8) at ./src/outguess.c:165 #3 0x000055771c8b14ba in steg_embedchunk (bitmap=bitmap@entry=0x7ffc6a269470, iter=iter@entry=0x7ffc6a268eb0, data=0, bits=1, bits@entry=8, embed=embed@entry=8) at ./src/outguess.c:206 #4 0x000055771c8b1aa8 in steg_embed (bitmap=bitmap@entry=0x7ffc6a269470, iter=iter@entry=0x7ffc6a268eb0, as=as@entry=0x7ffc6a268da0, data=data@entry=0x55771e27fb30 "BU\026\330E\315>ۆL\260\246\305\177", datalen=datalen@entry=12, seed=seed@entry=0, embed=8) at ./src/outguess.c:285 #5 0x000055771c8b1e44 in steg_find (bitmap=bitmap@entry=0x7ffc6a269470, iter=iter@entry=0x7ffc6a269270, as=as@entry=0x7ffc6a269050, siter=<optimized out>, siterstart=<optimized out>, data=data@entry=0x55771e27fb30 "BU\026\330E\315>ۆL\260\246\305\177", datalen=12, flags=8) at ./src/outguess.c:444 #6 0x000055771c8b2a04 in do_embed (bitmap=bitmap@entry=0x7ffc6a269470, filename=filename@entry=0x7ffc6a26b5cd "msg2.txt", key=key@entry=0x7ffc6a269820 "key2", klen=<optimized out>, cfg=cfg@entry=0x7ffc6a269464, result=result@entry=0x7ffc6a269448) at ./src/outguess.c:719 #7 0x000055771c8b0d12 in main (argc=<optimized out>, argv=0x7ffc6a269cc8) at ./src/outguess.c:1025 (rr) up #1 0x000055771c8b1301 in memmove (__len=12, __src=<optimized out>, __dest=0x7ffc6a268be0) at /usr/include/x86_64-linux-gnu/bits/string_fortified.h:36 warning: Source file is more recent than executable. 36 return __builtin___memmove_chk (__dest, __src, __len, (rr) up #2 steg_adjust_errors (bitmap=bitmap@entry=0x7ffc6a269470, flags=flags@entry=8) at ./src/outguess.c:165 165 memmove(detect + n + 1, detect + n, (rr) print detect $1 = {0, 0, 0} (rr) print &detect[0] $2 = (int *) 0x7ffc6a268bdc (rr) print &detect[1] $3 = (int *) 0x7ffc6a268be0 (rr) print n $4 = <optimized out> (rr) print sizeof(detect) $5 = 12 164 if (n < j - 1) { 165 memmove(detect + n + 1, detect + n, 166 (j - n) * sizeof(int)); 167 memmove(priority + n + 1, priority + n, 168 (j - n) * sizeof(int)); 169 } cd /home/benutzer/source/outguess cp -a orig try1 cd try1/outguess-0.4 DEB_BUILD_OPTIONS="noopt nostrip" dpkg-buildpackage -b dpkg --purge outguess-dbgsym dpkg -i /home/benutzer/source/outguess/try1/*.deb rr record outguess -k "key1" -d msg1.txt -E -K "key2" -D msg2.txt -p 100 JPEG_example_flower.jpg JPEG_example_flower.steg.jpg rr: Saving execution to trace directory `/home/benutzer/.local/share/rr/outguess-1'. cd /home/benutzer/source/outguess/try1/outguess-0.4 rr replay --debugger-option=-q outguess-1 set width 0 set pagination off display/i $pc cont b steg_adjust_errors reverse-cont reverse-cont Continuing. Breakpoint 1, steg_adjust_errors (bitmap=0x7ffd2c2330d0, flags=8) at ./src/outguess.c:140 140 { 1: x/i $pc => 0x55d1ce5a1518 <steg_adjust_errors+15>: mov %fs:0x28,%rax (rr) stepi 0x000055d1ce5a1521 140 { 1: x/i $pc => 0x55d1ce5a1521 <steg_adjust_errors+24>: mov %rax,-0x8(%rbp) (rr) x/1xg $rbp - 0x8 0x7ffd2c232818: 0x00007ffd2c232ad0 (rr) stepi 0x000055d1ce5a1525 140 { 1: x/i $pc => 0x55d1ce5a1525 <steg_adjust_errors+28>: xor %eax,%eax (rr) x/1xg $rbp - 0x8 0x7ffd2c232818: 0x82be3b60206e5f00 (rr) watch *0x7ffd2c232818 Hardware watchpoint 2: *0x7ffd2c232818 (rr) cont Continuing. Hardware watchpoint 2: *0x7ffd2c232818 Old value = 544104192 New value = 0 __memcpy_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:376 376 ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S: Datei oder Verzeichnis nicht gefunden. 1: x/i $pc => 0x7fcef07c4610 <__memcpy_avx_unaligned_erms+144>: ret (rr) bt #0 __memcpy_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:376 #1 0x000055d1ce5a16fc in steg_adjust_errors (bitmap=0x7ffd2c2330d0, flags=8) at ./src/outguess.c:165 #2 0x000055d1ce5a19cb in steg_embedchunk (bitmap=0x7ffd2c2330d0, iter=0x7ffd2c232ad0, data=0, bits=1, embed=8) at ./src/outguess.c:206 #3 0x000055d1ce5a1dc9 in steg_embed (bitmap=0x7ffd2c2330d0, iter=0x7ffd2c232ad0, as=0x7ffd2c2329c0, data=0x55d1ced53b30 "BU\026\330E\315>ۆL\260\246\316\177", datalen=12, seed=0, embed=8) at ./src/outguess.c:285 #4 0x000055d1ce5a24d6 in steg_find (bitmap=0x7ffd2c2330d0, iter=0x7ffd2c232e90, as=0x7ffd2c232c70, siter=256, siterstart=0, data=0x55d1ced53b30 "BU\026\330E\315>ۆL\260\246\316\177", datalen=12, flags=8) at ./src/outguess.c:444 #5 0x000055d1ce5a3323 in do_embed (bitmap=0x7ffd2c2330d0, filename=0x7ffd2c2355cd "msg2.txt", key=0x7ffd2c233480 "key2", klen=4, cfg=0x7ffd2c2330c4, result=0x7ffd2c2330ac) at ./src/outguess.c:719 #6 0x000055d1ce5a4089 in main (argc=2, argv=0x7ffd2c2338f8) at ./src/outguess.c:1025 (rr) up #1 0x000055d1ce5a16fc in steg_adjust_errors (bitmap=0x7ffd2c2330d0, flags=8) at ./src/outguess.c:165 165 memmove(detect + n + 1, detect + n, (rr) list 160 for (i = j; i < steg_err_cnt; i++) { 161 for (n = 0; n < j; n++) 162 if (detect[n] < bitmap->detect[steg_err_buf[i]]) 163 break; 164 if (n < j - 1) { 165 memmove(detect + n + 1, detect + n, 166 (j - n) * sizeof(int)); 167 memmove(priority + n + 1, priority + n, 168 (j - n) * sizeof(int)); 169 } (rr) print sizeof detect $1 = 12 (rr) print detect $2 = {0, 0, 0} (rr) print &detect[0] $3 = (int *) 0x7ffd2c23280c (rr) print &detect[1] $4 = (int *) 0x7ffd2c232810 (rr) print &detect[2] $5 = (int *) 0x7ffd2c232814 (rr) print n $6 = 0 (rr) print j $7 = 3 (rr) print (j - n) * sizeof(int) $8 = 12
