Byte swap was applied in the wrong order with testing for RLIM_INFINITY. On mips bigendian from an amd64 system this results in infinity being misinterpretted as 2^31-1.
This is a serious bug because it causes setrlimit stack size to kill all child processes. This means (for example) that 'make' can run no children. The mechanism of failure: 1. parent sets stack size rlimit to 'infinity' 2. qemu screws this value up 3. child process fetches stack size as a large (but non-infinite) value 4. qemu tries to allocate stack before execution 5. stack allocation fails (too big) and child process dies Signed-off-by: Wesley W. Terpstra <terps...@debian.org> --- diff --git a/linux-user/syscall.c b/linux-user/syscall.c index fed7a8f..0b0a3d0 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -919,18 +919,30 @@ static inline abi_long host_to_target_rusage(abi_ulong target_addr, static inline rlim_t target_to_host_rlim(target_ulong target_rlim) { - if (target_rlim == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; + target_ulong target_rlim_swap; + rlim_t result; + + target_rlim_swap = tswapl(target_rlim); + if (target_rlim_swap == TARGET_RLIM_INFINITY || target_rlim_swap != (rlim_t)target_rlim_swap) + result = RLIM_INFINITY; else - return tswapl(target_rlim); + result = target_rlim_swap; + + return result; } static inline target_ulong host_to_target_rlim(rlim_t rlim) { + target_ulong target_rlim_swap; + target_ulong result; + if (rlim == RLIM_INFINITY || rlim != (target_long)rlim) - return TARGET_RLIM_INFINITY; + target_rlim_swap = TARGET_RLIM_INFINITY; else - return tswapl(rlim); + target_rlim_swap = rlim; + result = tswapl(target_rlim_swap); + + return result; } static inline abi_long copy_from_user_timeval(struct timeval *tv,
diff --git a/linux-user/syscall.c b/linux-user/syscall.c index fed7a8f..0b0a3d0 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -919,18 +919,30 @@ static inline abi_long host_to_target_rusage(abi_ulong target_addr, static inline rlim_t target_to_host_rlim(target_ulong target_rlim) { - if (target_rlim == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; + target_ulong target_rlim_swap; + rlim_t result; + + target_rlim_swap = tswapl(target_rlim); + if (target_rlim_swap == TARGET_RLIM_INFINITY || target_rlim_swap != (rlim_t)target_rlim_swap) + result = RLIM_INFINITY; else - return tswapl(target_rlim); + result = target_rlim_swap; + + return result; } static inline target_ulong host_to_target_rlim(rlim_t rlim) { + target_ulong target_rlim_swap; + target_ulong result; + if (rlim == RLIM_INFINITY || rlim != (target_long)rlim) - return TARGET_RLIM_INFINITY; + target_rlim_swap = TARGET_RLIM_INFINITY; else - return tswapl(rlim); + target_rlim_swap = rlim; + result = tswapl(target_rlim_swap); + + return result; } static inline abi_long copy_from_user_timeval(struct timeval *tv,