New crash, same thing with ipipe panic trace (the decoded log does not add information to the relevant parts).
Is the dump_stack function itself trashing the stack? [ 168.411205] [Xenomai] watchdog triggered on CPU #1 -- runaway thread 'main' signaled [ 209.176742] ------------[ cut here ]------------ [ 209.181381] xnthread_relax() failed for thread aboard_runner[790] [ 209.181389] BUG: Unhandled exception over domain Xenomai at 0xffffffff99997fed - switching to ROOT [ 209.196451] CPU: 0 PID: 790 Comm: aboard_runner Tainted: G W 4.19.84-xenod8-static #1 [ 209.205588] Hardware name: TQ-Group TQMxE39M/Type2 - Board Product Name, BIOS 5.12.30.21.16 01/31/2019 [ 209.214900] I-pipe domain: Linux [ 209.218137] Call Trace: [ 209.220593] dump_stack+0x8c/0xc0 [ 209.223919] __ipipe_trap_prologue.cold+0x1f/0x5e [ 209.228629] invalid_op+0x26/0x51 [ 209.231952] RIP: 0010:xnthread_relax+0x46d/0x4a0 [ 209.236576] Code: f6 83 c2 11 00 00 01 75 0e 48 8b 03 48 85 c0 74 33 8b 90 c0 04 00 00 48 8d b3 5c 14 00 00 48 c7 c7 90 00 8b 9a e8 02 02 ef ff <0f> 0b e9 42 fd ff ff 89 c6 48 c7 c7 c4 f8 a3 9a e8 2e 71 f3 ff e9 [ 209.255347] RSP: 0018:ffff9a0e4074fd90 EFLAGS: 00010286 [ 209.260586] RAX: 0000000000000000 RBX: ffff9a0e4065aa40 RCX: 000000000000000b [ 209.267728] RDX: 0000000000005129 RSI: ffff902a794791f8 RDI: 00000000007800c0 [ 209.274869] RBP: ffff9a0e4074fe68 R08: 00000000007800c0 R09: 000000000002e248 [ 209.282013] R10: ffffffff9bb72040 R11: ffffffff9bb3209c R12: ffffffff9bbfdc80 [ 209.289157] R13: ffff902a76da8000 R14: 0000000000000001 R15: 0000000000000292 [ 209.296299] ? xnthread_prepare_wait+0x20/0x20 [ 209.300752] ? trace+0x59/0x8d [ 209.303814] ? __cobalt_clock_nanosleep+0x540/0x540 [ 209.308700] handle_head_syscall+0x307/0x370 [ 209.312979] ipipe_fastcall_hook+0x14/0x20 [ 209.317083] ipipe_handle_syscall+0x57/0xe0 [ 209.321280] do_syscall_64+0x4b/0x500 [ 209.324950] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 209.330011] RIP: 0033:0x7ffff7f9bd68 [ 209.333598] Code: 89 fb bf 01 00 00 00 48 83 ec 18 48 8d 74 24 0c e8 bd f3 ff ff b9 19 00 00 10 48 63 f5 48 63 fb 4d 89 ea 4c 89 e2 89 c8 0f 05 <8b> 7c 24 0c 48 89 c3 31 f6 e8 9a f3 ff ff 48 83 c4 18 89 d8 f7 d8 [ 209.352370] RSP: 002b:00007fffffffe7d0 EFLAGS: 00000246 ORIG_RAX: 0000000010000019 [ 209.359954] RAX: fffffffffffffe00 RBX: 0000000000000001 RCX: 00007ffff7f9bd68 [ 209.367098] RDX: 00007fffffffe820 RSI: 0000000000000001 RDI: 0000000000000001 [ 209.374237] RBP: 0000000000000001 R08: 0000000000000001 R09: 0000000000000014 [ 209.381381] R10: 00007fffffffe820 R11: 0000000000000246 R12: 00007fffffffe820 [ 209.388524] R13: 00007fffffffe820 R14: 0000000000000000 R15: 0000000000000000 [ 209.395665] I-pipe tracer log (100 points): [ 209.399857] | #func 0 ipipe_trace_panic_freeze+0x0 (__ipipe_trap_prologue+0x237) [ 209.409056] | +func 0 ipipe_root_only+0x0 (ipipe_stall_root+0xe) [ 209.416862] | +func -1 ipipe_stall_root+0x0 (__ipipe_trap_prologue+0x2ae) [ 209.425365] |+ func -2 ipipe_trap_hook+0x0 (__ipipe_notify_trap+0x98) [ 209.433523] |+ func -3 __ipipe_notify_trap+0x0 (__ipipe_trap_prologue+0x7f) [ 209.442199] |+ func -4 __ipipe_trap_prologue+0x0 (invalid_op+0x26) [ 209.450097] |+ end 0x80000001 -5 __ipipe_spin_unlock_irqrestore+0x4f (<00000000>) [ 209.458425] |# func -6 __ipipe_spin_unlock_irqrestore+0x0 (__ipipe_log_printk+0x69) [ 209.467797] |+ begin 0x80000001 -10 __ipipe_spin_lock_irqsave+0x5e (<00000000>) [ 209.475693] + func -10 __ipipe_spin_lock_irqsave+0x0 (__ipipe_log_printk+0x22) [ 209.484630] + func -10 __ipipe_log_printk+0x0 (__warn_printk+0x6c) [ 209.492525] |+ end 0x80000001 -11 do_vprintk+0xf6 (<00000000>) [ 209.499120] |+ begin 0x80000001 -11 do_vprintk+0x106 (<00000000>) [ 209.505799] + func -12 do_vprintk+0x0 (__warn_printk+0x6c) [ 209.513000] + func -12 vprintk+0x0 (__warn_printk+0x6c) [ 209.519939] |+ end 0x80000001 -12 ipipe_raise_irq+0x70 (<00000000>) [ 209.526969] |+ func -13 __ipipe_set_irq_pending+0x0 (__ipipe_dispatch_irq+0xad) [ 209.535905] |+ func -14 __ipipe_dispatch_irq+0x0 (ipipe_raise_irq+0x7e) [ 209.544148] |+ begin 0x80000001 -14 ipipe_raise_irq+0x64 (<00000000>) [ 209.551178] + func -15 ipipe_raise_irq+0x0 (__ipipe_log_printk+0x84) [ 209.559250] |+ end 0x80000001 -15 __ipipe_spin_unlock_irqrestore+0x4f (<00000000>) [ 209.567581] |# func -15 __ipipe_spin_unlock_irqrestore+0x0 (__ipipe_log_printk+0x69) [ 209.576951] |+ begin 0x80000001 -17 __ipipe_spin_lock_irqsave+0x5e (<00000000>) [ 209.584847] + func -18 __ipipe_spin_lock_irqsave+0x0 (__ipipe_log_printk+0x22) [ 209.593784] + func -18 __ipipe_log_printk+0x0 (printk+0x58) [ 209.601073] |+ end 0x80000001 -19 do_vprintk+0xf6 (<00000000>) [ 209.607667] |+ begin 0x80000001 -20 do_vprintk+0x106 (<00000000>) [ 209.614350] + func -20 do_vprintk+0x0 (printk+0x58) [ 209.620943] + func -21 printk+0x0 (__warn_printk+0x47) [ 209.627794] + func -21 __warn_printk+0x0 (xnthread_relax+0x46d) [ 209.635429] |+ end 0x80000001 -22 xnthread_relax+0x3e6 (<00000000>) [ 209.642460] |+ begin 0x80000001 -22 xnthread_relax+0x3cc (<00000000>) [ 209.649487] |+ end 0x80000000 -23 ipipe_unstall_head+0x4a (<00000000>) [ 209.656776] |# func -23 ipipe_unstall_head+0x0 (xnthread_relax+0x186) [ 209.664842] |# [ 790] aboard_ -1 -24 ___xnsched_run.part.0+0x2a5 (<00000000>) [ 209.672480] |# func -24 xnarch_switch_fpu+0x0 (xnthread_switch_fpu+0x26) [ 209.680809] |# func -24 xnthread_switch_fpu+0x0 (___xnsched_run.part.0+0x28a) [ 209.689576] |# func -25 switch_mm_irqs_off+0x0 (xnarch_switch_to+0x2b) [ 209.697736] |# func -26 xnarch_switch_to+0x0 (___xnsched_run.part.0+0x228) [ 209.706245] |# func -26 xnclock_core_local_shot+0x0 (___xnsched_run.part.0+0x468) [ 209.715363] |# func -27 xntimer_heading_p+0x0 (__xntimer_stop+0xb8) [ 209.723261] |# func -27 __xntimer_stop+0x0 (___xnsched_run.part.0+0x468) [ 209.731589] |# func -28 xnsched_idle_pick+0x0 (xnsched_pick_next+0x7d) [ 209.739747] |# func -28 xnsched_tp_pick+0x0 (xnsched_pick_next+0x7d) [ 209.747733] |# func -29 xnsched_rt_pick+0x0 (xnsched_pick_next+0x7d) [ 209.755721] |# func -29 xnsched_pick_next+0x0 (___xnsched_run.part.0+0x173) [ 209.764315] |# [ 790] aboard_r 1 -30 ___xnsched_run.part.0+0x95 (<00000000>) [ 209.771862] |# func -30 ___xnsched_run.part.0+0x0 (xnthread_relax+0x181) [ 209.780195] |# func -31 ___xnsched_run+0x0 (xnthread_relax+0x181) [ 209.787914] |+ begin 0x80000000 -31 xnthread_suspend+0x508 (<00000000>) [ 209.795115] |+ end 0x80000000 -32 __ipipe_restore_head+0x4f (<00000000>) [ 209.802577] |# func -32 __ipipe_restore_head+0x0 (xnthread_suspend+0x515) [ 209.810996] |# func -33 xnthread_suspend+0x0 (xnthread_relax+0x181) [ 209.818893] |# func -34 __ipipe_set_irq_pending+0x0 (__ipipe_post_work_root+0xac) [ 209.828004] |# func -35 __ipipe_post_work_root+0x0 (xnthread_relax+0xcf) [ 209.836331] |+ begin 0x80000000 -35 xnthread_relax+0x3af (<00000000>) [ 209.843361] |+ end 0x80000001 -36 xnthread_relax+0x362 (<00000000>) [ 209.850392] |+ begin 0x80000001 -36 xnthread_relax+0x339 (<00000000>) [ 209.857420] + func -37 xnthread_relax+0x0 (handle_head_syscall+0x307) [ 209.865573] |+ end 0x80000000 -37 __ipipe_restore_head+0x4f (<00000000>) [ 209.873032] |# func -38 __ipipe_restore_head+0x0 (prepare_for_signal+0x131) [ 209.881622] |+ begin 0x80000000 -38 prepare_for_signal+0x124 (<00000000>) [ 209.889001] + func -39 prepare_for_signal+0x0 (handle_head_syscall+0x307) [ 209.897507] |+ end 0x80000000 -39 __ipipe_restore_head+0x4f (<00000000>) [ 209.904969] |# func -40 __ipipe_restore_head+0x0 (__cobalt_clock_nanosleep+0x3fb) [ 209.914079] |# func -41 xnthread_suspend+0x0 (__cobalt_clock_nanosleep+0x11c) [ 209.922842] |+ begin 0x80000000 -42 __cobalt_clock_nanosleep+0x25a (<00000000>) [ 209.930742] + func -42 __cobalt_clock_nanosleep+0x0 (CoBaLt_clock_nanosleep+0x7f) [ 209.939938] + func -42 CoBaLt_clock_nanosleep+0x0 (handle_head_syscall+0xe1) [ 209.948704] + func -43 handle_head_syscall+0x0 (ipipe_fastcall_hook+0x14) [ 209.957208] + func -43 ipipe_fastcall_hook+0x0 (ipipe_handle_syscall+0x57) [ 209.965798] + func -44 ipipe_handle_syscall+0x0 (do_syscall_64+0x4b) [ 209.973867] |+ end 0x00001101 -44 do_syscall_64+0x34 (<00000000>) [ 209.980724] |# [ 790] aboard_r 1 -45 ___xnsched_run.part.0+0x2a5 (<00000000>) [ 209.988360] |# func -46 xnarch_switch_fpu+0x0 (xnthread_switch_fpu+0x26) [ 209.996690] |# func -46 xnthread_switch_fpu+0x0 (___xnsched_run.part.0+0x28a) [ 210.005454] |# func -48 switch_mm_irqs_off+0x0 (xnarch_switch_to+0x2b) [ 210.006879] ------------[ cut here ]------------ [ 210.013612] | [ 210.018242] do_IRQ(): swapper/1 has overflown the kernel stack (cur:ffff9a0e400a4000,sp:ffffffff9aa03db0,irq stk top-bottom:ffff902a79800080-ffff902a79804000,exception stk top-bottom:fffffe0000036080-fffffe0000038000,ip) [ 210.018247] # [ 210.019990] Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: __schedule+0x8c5/0x8d0 [ 210.019990] [ 210.041893] [ 210.043480] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 4.19.84-xenod8-static #1 [ 210.054867] func [ 210.056453] Hardware name: TQ-Group TQMxE39M/Type2 - Board Product Name, BIOS 5.12.30.21.16 01/31/2019 [ 210.065064] [ 210.067258] I-pipe domain: Linux [ 210.076563] -49 [ 210.079018] Call Trace: [ 210.082246] xnarch_switch_to+0x0 (___xnsched_run.part.0+0x228) [ 210.092655] |# func -50 lapic_next_deadline+0x0 (ipipe_timer_set+0x58) [ 210.100811] |# func -50 ipipe_timer_set+0x0 (xntimer_start+0x10f) [ 210.108535] |# event tick@1305312 -51 xnclock_core_local_shot+0x77 (<00000000>) [ 210.116348] |# func -51 xnclock_core_local_shot+0x0 (xntimer_start+0x10f) [ 210.124763] |# func -51 xntimer_heading_p+0x0 (xntimer_enqueue_and_program+0x2a) [ 210.133789] |# func -52 xntimerq_insert+0x0 (xntimer_enqueue_and_program+0x15) [ 210.142641] |# func -53 xntimer_enqueue_and_program+0x0 (xntimer_start+0x10f) [ 210.151406] |# func -53 xnclock_core_ns_to_ticks+0x0 (xntimer_start+0x24b) [ 210.159912] |# func -54 xntimer_start+0x0 (___xnsched_run.part.0+0x389) [ 210.168156] |# func -55 xnarch_leave_root+0x0 (___xnsched_run.part.0+0x366) [ 210.176744] |# func -55 __ipipe_notify_vm_preemption+0x0 (___xnsched_run.part.0+0x329) [ 210.186287] |# func -56 xnsched_rt_pick+0x0 (xnsched_pick_next+0x7d) [ 210.194267] |# func -57 xnsched_pick_next+0x0 (___xnsched_run.part.0+0x173) [ 210.202857] |# [ 790] aboard_ -1 -57 ___xnsched_run.part.0+0x95 (<00000000>) [ 210.210408] |# func -58 ___xnsched_run.part.0+0x0 (dispatch_irq_head+0x95) [ 210.218910] |# func -59 __xnsched_run_handler+0x0 (dispatch_irq_head+0x95) [ 210.227411] |+ func -60 dispatch_irq_head+0x0 (__ipipe_handle_irq+0xa0) [ 210.235656] |+ func -60 __ipipe_ack_apic+0x0 (__ipipe_dispatch_irq+0x146) [ 210.244073] |+ func -62 irq_to_desc+0x0 (__ipipe_dispatch_irq+0x37) [ 210.251972] |+ func -62 __ipipe_dispatch_irq+0x0 (__ipipe_handle_irq+0xa0) [ 210.260475] |+ begin 0x00001101 -62 do_syscall_64+0x34 (<00000000>) [ 210.267330] |+ func -63 __ipipe_handle_irq+0x0 (ipipe_reschedule_interrupt+0x12) [ 210.276354] WARNING: CPU: 0 PID: 790 at kernel/xenomai/thread.c:2124 xnthread_relax+0x46d/0x4a0 [ 210.285058] Modules linked in: rt_igb plusb usbnet mii [ 210.290238] CPU: 0 PID: 790 Comm: aboard_runner Tainted: G W 4.19.84-xenod8-static #1 [ 210.299377] Hardware name: TQ-Group TQMxE39M/Type2 - Board Product Name, BIOS 5.12.30.21.16 01/31/2019 [ 210.308689] I-pipe domain: Linux [ 210.311928] RIP: 0010:xnthread_relax+0x46d/0x4a0 [ 210.316552] Code: f6 83 c2 11 00 00 01 75 0e 48 8b 03 48 85 c0 74 33 8b 90 c0 04 00 00 48 8d b3 5c 14 00 00 48 c7 c7 90 00 8b 9a e8 02 02 ef ff <0f> 0b e9 42 fd ff ff 89 c6 48 c7 c7 c4 f8 a3 9a e8 2e 71 f3 ff e9 [ 210.335323] RSP: 0018:ffff9a0e4074fd90 EFLAGS: 00010286 [ 210.340561] RAX: 0000000000000000 RBX: ffff9a0e4065aa40 RCX: 000000000000000b [ 210.347705] RDX: 0000000000005129 RSI: ffff902a794791f8 RDI: 00000000007800c0 [ 210.354847] RBP: ffff9a0e4074fe68 R08: 00000000007800c0 R09: 000000000002e248 [ 210.361989] R10: ffffffff9bb72040 R11: ffffffff9bb3209c R12: ffffffff9bbfdc80 [ 210.369129] R13: ffff902a76da8000 R14: 0000000000000001 R15: 0000000000000292 [ 210.376269] FS: 00007ffff7a327c0(0000) GS:ffff902a78c00000(0000) knlGS:0000000000000000 [ 210.384366] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 210.390119] CR2: 00000000015f07b8 CR3: 0000000173042000 CR4: 00000000003406f0 [ 210.397263] Call Trace: [ 210.399719] ? xnthread_prepare_wait+0x20/0x20 [ 210.404169] ? trace+0x59/0x8d [ 210.407231] ? __cobalt_clock_nanosleep+0x540/0x540 [ 210.412118] handle_head_syscall+0x307/0x370 [ 210.416397] ipipe_fastcall_hook+0x14/0x20 [ 210.420499] ipipe_handle_syscall+0x57/0xe0 [ 210.424690] do_syscall_64+0x4b/0x500 [ 210.428361] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 210.433421] RIP: 0033:0x7ffff7f9bd68 [ 210.437006] Code: 89 fb bf 01 00 00 00 48 83 ec 18 48 8d 74 24 0c e8 bd f3 ff ff b9 19 00 00 10 48 63 f5 48 63 fb 4d 89 ea 4c 89 e2 89 c8 0f 05 <8b> 7c 24 0c 48 89 c3 31 f6 e8 9a f3 ff ff 48 83 c4 18 89 d8 f7 d8 [ 210.455780] RSP: 002b:00007fffffffe7d0 EFLAGS: 00000246 ORIG_RAX: 0000000010000019 [ 210.463364] RAX: fffffffffffffe00 RBX: 0000000000000001 RCX: 00007ffff7f9bd68 [ 210.470509] RDX: 00007fffffffe820 RSI: 0000000000000001 RDI: 0000000000000001 [ 210.477650] RBP: 0000000000000001 R08: 0000000000000001 R09: 0000000000000014 [ 210.484793] R10: 00007fffffffe820 R11: 0000000000000246 R12: 00007fffffffe820 [ 210.491933] R13: 00007fffffffe820 R14: 0000000000000000 R15: 0000000000000000 [ 210.499074] ---[ end trace cd327088d6cd89c6 ]--- [ 210.503698] WARNING: CPU: 1 PID: 0 at arch/x86/kernel/irq_64.c:72 handle_irq+0xec/0xf4 [ 210.511621] Modules linked in: rt_igb plusb usbnet mii [ 210.516798] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 4.19.84-xenod8-static #1 [ 210.525417] Hardware name: TQ-Group TQMxE39M/Type2 - Board Product Name, BIOS 5.12.30.21.16 01/31/2019 [ 210.534728] I-pipe domain: Linux [ 210.537966] RIP: 0010:handle_irq+0xec/0xf4 [ 210.542072] Code: 25 40 4c 01 00 49 81 c2 68 06 00 00 c6 05 06 34 2c 01 01 ff b6 80 00 00 00 4c 89 d6 50 57 48 c7 c7 a0 eb 88 9a e8 d3 1b 07 00 <0f> 0b 48 83 c4 18 eb b3 48 c7 c7 28 ec 88 9a e8 48 1c 07 00 e8 9b [ 210.560841] RSP: 0018:ffffffff9aa03d98 EFLAGS: 00010082 [ 210.566080] RAX: 0000000000000000 RBX: ffff902a76deea00 RCX: 0000000000000006 [ 210.573220] RDX: 0000000000000007 RSI: 0000000000000000 RDI: ffff902a798153f0 [ 210.580364] RBP: ffff902a7982e128 R08: ffffffff9bb3184d R09: 0000000000000004 [ 210.587507] R10: 0000000000000007 R11: ffffffff9bb3184d R12: ffff902a76deea00 [ 210.594649] R13: 0000000000000023 R14: 0000000000000000 R15: 0000000000000002 [ 210.601795] FS: 0000000000000000(0000) GS:ffff902a79800000(0000) knlGS:0000000000000000 [ 210.609892] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 210.615645] CR2: 000000000053735c CR3: 0000000173042000 CR4: 00000000003406e0 [ 210.622785] Call Trace: [ 210.625239] ---[ end trace cd327088d6cd89c7 ]--- [ 210.629883] Kernel Offset: 0x18800000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) [ 210.640695] ---[ end Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: __schedule+0x8c5/0x8d0 [ 210.640695] ]--- ________________________________ This message and any attachments are solely for the use of the intended recipients. They may contain privileged and/or confidential information or other information protected from disclosure. If you are not an intended recipient, you are hereby notified that you received this email in error and that any review, dissemination, distribution or copying of this email and any attachment is strictly prohibited. If you have received this email in error, please contact the sender and delete the message and any attachment from your system. ANDRITZ HYDRO GmbH Rechtsform/ Legal form: Gesellschaft mit beschränkter Haftung / Corporation Firmensitz/ Registered seat: Wien Firmenbuchgericht/ Court of registry: Handelsgericht Wien Firmenbuchnummer/ Company registration: FN 61833 g DVR: 0605077 UID-Nr.: ATU14756806 Thank You ________________________________
