Negating the smallest negative value of a signed type overflows, which
is undefined behavior. The selftests are built with:
-fsanitize=undefined -fsanitize-trap=all
so a caller passing INT_MIN does not merely get an unspecified answer,
it dies (on both x86-64 and i386):
A simple test program:
printf("x = %d\n", abs(INT_MIN));
$ ./ab
Illegal instruction (core dumped)
(gdb) bt
#0 0x0000000000401009 in main ()
(gdb) x/6i main
0x401000 <main>: mov $0x80000000,%eax
0x401005 <main+5>: neg %eax
0x401007 <main+7>: jno 0x40100b <main+11>
=> 0x401009 <main+9>: ud2
0x40100b <main+11>: push %rax
0x40100c <main+12>: mov $0x80000000,%esi
Negate in the corresponding unsigned type instead. The value still
cannot be represented in the result type, so the minimum is returned
unchanged.
Cc: Yichun Zhang <[email protected]>
Cc: Alviro Iskandar Setiawan <[email protected]>
Fixes: bf5e8a78bede ("tools/nolibc: add abs() and friends")
Signed-off-by: Ammar Faizi <[email protected]>
---
tools/include/nolibc/stdlib.h | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/tools/include/nolibc/stdlib.h b/tools/include/nolibc/stdlib.h
index 1816c2368b68..8d86044f759f 100644
--- a/tools/include/nolibc/stdlib.h
+++ b/tools/include/nolibc/stdlib.h
@@ -32,22 +32,28 @@ static __attribute__((unused)) char itoa_buffer[21];
* As much as possible, please keep functions alphabetically sorted.
*/
+/*
+ * The absolute value of the smallest negative value is not representable in
+ * the result type. Negate in the unsigned type so that the overflow is
+ * defined and return it unchanged, like the other libcs do.
+ */
+
static __inline__
int abs(int j)
{
- return j >= 0 ? j : -j;
+ return j >= 0 ? j : (int)-(unsigned int)j;
}
static __inline__
long labs(long j)
{
- return j >= 0 ? j : -j;
+ return j >= 0 ? j : (long)-(unsigned long)j;
}
static __inline__
long long llabs(long long j)
{
- return j >= 0 ? j : -j;
+ return j >= 0 ? j : (long long)-(unsigned long long)j;
}
/* must be exported, as it's used by libgcc for various divide functions */
--
Ammar Faizi
